1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.test_mutant.schemata;
11 
12 import logger = std.experimental.logger;
13 import std.algorithm : sort, map;
14 import std.array : empty;
15 import std.conv : to;
16 import std.datetime : Duration;
17 import std.exception : collectException;
18 import std.typecons : Tuple;
19 
20 import proc : DrainElement;
21 import sumtype;
22 
23 import my.fsm : Fsm, next, act, get, TypeDataMap;
24 static import my.fsm;
25 
26 import dextool.plugin.mutate.backend.database : MutationStatusId, Database, spinSql;
27 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
28 import dextool.plugin.mutate.backend.test_mutant.common;
29 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
30 import dextool.plugin.mutate.backend.type : Mutation, TestCase, Checksum;
31 import dextool.plugin.mutate.type : TestCaseAnalyzeBuiltin, ShellCommand;
32 
33 @safe:
34 
35 struct MutationTestResult {
36     import std.datetime : Duration;
37     import dextool.plugin.mutate.backend.database : MutationStatusId;
38     import dextool.plugin.mutate.backend.type : TestCase;
39 
40     MutationStatusId id;
41     Mutation.Status status;
42     Duration testTime;
43     TestCase[] testCases;
44 }
45 
46 struct SchemataTestDriver {
47     private {
48         /// True as long as the schemata driver is running.
49         bool isRunning_ = true;
50 
51         FilesysIO fio;
52 
53         Database* db;
54 
55         /// Runs the test commands.
56         TestRunner* runner;
57 
58         /// Result of testing the mutants.
59         MutationTestResult[] result_;
60     }
61 
62     static struct None {
63     }
64 
65     static struct InitializeData {
66         MutationStatusId[] mutants;
67     }
68 
69     static struct Initialize {
70     }
71 
72     static struct Done {
73     }
74 
75     static struct NextMutantData {
76         /// Mutants to test.
77         InjectIdResult mutants;
78     }
79 
80     static struct NextMutant {
81         bool done;
82         InjectIdResult.InjectId inject;
83     }
84 
85     static struct TestMutantData {
86         /// If the user has configured that the test cases should be analyzed.
87         bool hasTestCaseOutputAnalyzer;
88     }
89 
90     static struct TestMutant {
91         InjectIdResult.InjectId inject;
92 
93         MutationTestResult result;
94         bool hasTestOutput;
95         // if there are mutants status id's related to a file but the mutants
96         // have been removed.
97         bool mutantIdError;
98     }
99 
100     static struct TestCaseAnalyzeData {
101         TestCaseAnalyzer* testCaseAnalyzer;
102         DrainElement[] output;
103     }
104 
105     static struct TestCaseAnalyze {
106         MutationTestResult result;
107         bool unstableTests;
108     }
109 
110     static struct StoreResult {
111         MutationTestResult result;
112     }
113 
114     alias Fsm = my.fsm.Fsm!(None, Initialize, Done, NextMutant, TestMutant,
115             TestCaseAnalyze, StoreResult);
116     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData,
117             NextMutantData, InitializeData);
118 
119     private {
120         Fsm fsm;
121         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze, NextMutant, Initialize) local;
122     }
123 
124     this(FilesysIO fio, TestRunner* runner, Database* db,
125             TestCaseAnalyzer* testCaseAnalyzer, MutationStatusId[] mutants) {
126         this.fio = fio;
127         this.runner = runner;
128         this.db = db;
129         this.local.get!Initialize.mutants = mutants;
130         this.local.get!TestCaseAnalyze.testCaseAnalyzer = testCaseAnalyzer;
131         this.local.get!TestMutant.hasTestCaseOutputAnalyzer = !testCaseAnalyzer.empty;
132     }
133 
134     static void execute_(ref SchemataTestDriver self) @trusted {
135         self.fsm.next!((None a) => fsm(Initialize.init),
136                 (Initialize a) => fsm(NextMutant.init), (NextMutant a) {
137             if (a.done)
138                 return fsm(Done.init);
139             return fsm(TestMutant(a.inject));
140         }, (TestMutant a) {
141             if (a.mutantIdError)
142                 return fsm(NextMutant.init);
143             if (a.result.status == Mutation.Status.killed
144                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) {
145                 return fsm(TestCaseAnalyze(a.result));
146             }
147             return fsm(StoreResult(a.result));
148         }, (TestCaseAnalyze a) {
149             if (a.unstableTests)
150                 return fsm(NextMutant.init);
151             return fsm(StoreResult(a.result));
152         }, (StoreResult a) => fsm(NextMutant.init), (Done a) => fsm(a));
153 
154         debug logger.trace("state: ", self.fsm.logNext);
155         self.fsm.act!(self);
156     }
157 
158 nothrow:
159 
160     MutationTestResult[] result() {
161         return result_;
162     }
163 
164     void execute() {
165         try {
166             execute_(this);
167         } catch (Exception e) {
168             logger.warning(e.msg).collectException;
169         }
170     }
171 
172     bool isRunning() {
173         return isRunning_;
174     }
175 
176     void opCall(None data) {
177     }
178 
179     void opCall(Initialize data) {
180         scope (exit)
181             local.get!Initialize.mutants = null;
182 
183         InjectIdBuilder builder;
184         foreach (mutant; local.get!Initialize.mutants) {
185             auto cs = spinSql!(() { return db.getChecksum(mutant); });
186             if (!cs.isNull) {
187                 builder.put(mutant, cs.get);
188             }
189         }
190         debug logger.trace(builder).collectException;
191 
192         local.get!NextMutant.mutants = builder.finalize;
193     }
194 
195     void opCall(Done data) {
196         isRunning_ = false;
197     }
198 
199     void opCall(ref NextMutant data) {
200         data.done = local.get!NextMutant.mutants.empty;
201 
202         if (!data.done) {
203             data.inject = local.get!NextMutant.mutants.front;
204             local.get!NextMutant.mutants.popFront;
205         }
206     }
207 
208     void opCall(ref TestMutant data) {
209         import std.datetime.stopwatch : StopWatch, AutoStart;
210         import dextool.plugin.mutate.backend.analyze.pass_schemata : schemataMutantEnvKey,
211             checksumToId;
212         import dextool.plugin.mutate.backend.generate_mutant : makeMutationText;
213 
214         data.result.id = data.inject.statusId;
215 
216         auto id = spinSql!(() { return db.getMutationId(data.inject.statusId); });
217         if (id.isNull) {
218             data.mutantIdError = true;
219             return;
220         }
221         auto entry_ = spinSql!(() { return db.getMutation(id.get); });
222         if (entry_.isNull) {
223             data.mutantIdError = true;
224             return;
225         }
226         auto entry = entry_.get;
227 
228         try {
229             const file = fio.toAbsoluteRoot(entry.file);
230             auto original = fio.makeInput(file);
231             auto txt = makeMutationText(original, entry.mp.offset,
232                     entry.mp.mutations[0].kind, entry.lang);
233             debug logger.trace(entry);
234             logger.infof("%s from '%s' to '%s' in %s:%s:%s", data.inject.injectId,
235                     txt.original, txt.mutation, file, entry.sloc.line, entry.sloc.column);
236         } catch (Exception e) {
237             logger.info(e.msg).collectException;
238         }
239 
240         runner.env[schemataMutantEnvKey] = data.inject.injectId.to!string;
241         scope (exit)
242             runner.env.remove(schemataMutantEnvKey);
243 
244         auto sw = StopWatch(AutoStart.yes);
245         auto res = runTester(*runner);
246         data.result.testTime = sw.peek;
247 
248         data.result.status = res.status;
249         data.hasTestOutput = !res.output.empty;
250         local.get!TestCaseAnalyze.output = res.output;
251 
252         logger.infof("%s %s (%s)", data.inject.injectId, data.result.status,
253                 data.result.testTime).collectException;
254     }
255 
256     void opCall(ref TestCaseAnalyze data) {
257         try {
258             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(
259                     local.get!TestCaseAnalyze.output);
260             local.get!TestCaseAnalyze.output = null;
261 
262             analyze.match!((TestCaseAnalyzer.Success a) {
263                 data.result.testCases = a.failed;
264             }, (TestCaseAnalyzer.Unstable a) {
265                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
266                 logger.info(
267                     "As configured the result is ignored which will force the mutant to be re-tested");
268                 data.unstableTests = true;
269             }, (TestCaseAnalyzer.Failed a) {
270                 logger.warning("The parser that analyze the output from test case(s) failed");
271             });
272 
273             logger.infof(!data.result.testCases.empty, `%s killed by [%-(%s, %)]`,
274                     data.result.id, data.result.testCases.sort.map!"a.name").collectException;
275         } catch (Exception e) {
276             logger.warning(e.msg).collectException;
277         }
278     }
279 
280     void opCall(StoreResult data) {
281         result_ ~= data.result;
282     }
283 }
284 
285 /** Generate schemata injection IDs (32bit) from mutant checksums (128bit).
286  *
287  * There is a possibility that an injection ID result in a collision because
288  * they are only 32 bit. If that happens the mutant is discarded as unfeasable
289  * to use for schemata.
290  *
291  * TODO: if this is changed to being order dependent then it can handle all
292  * mutants. But I can't see how that can be done easily both because of how the
293  * schemas are generated and how the database is setup.
294  */
295 struct InjectIdBuilder {
296     import my.set;
297 
298     private {
299         alias InjectId = InjectIdResult.InjectId;
300 
301         InjectId[uint] result;
302         Set!uint collisions;
303     }
304 
305     void put(MutationStatusId id, Checksum cs) @safe pure nothrow {
306         import dextool.plugin.mutate.backend.analyze.pass_schemata : checksumToId;
307 
308         const injectId = checksumToId(cs);
309         debug logger.tracef("%s %s %s", id, cs, injectId).collectException;
310 
311         if (injectId in collisions) {
312         } else if (injectId in result) {
313             collisions.add(injectId);
314             result.remove(injectId);
315         } else {
316             result[injectId] = InjectId(id, injectId);
317         }
318     }
319 
320     InjectIdResult finalize() @safe pure nothrow {
321         import std.array : array;
322 
323         return InjectIdResult(result.byValue.array);
324     }
325 }
326 
327 struct InjectIdResult {
328     alias InjectId = Tuple!(MutationStatusId, "statusId", uint, "injectId");
329     InjectId[] ids;
330 
331     InjectId front() @safe pure nothrow {
332         assert(!empty, "Can't get front of an empty range");
333         return ids[0];
334     }
335 
336     void popFront() @safe pure nothrow {
337         assert(!empty, "Can't pop front of an empty range");
338         ids = ids[1 .. $];
339     }
340 
341     bool empty() @safe pure nothrow const @nogc {
342         return ids.empty;
343     }
344 }
345 
346 @("shall detect a collision and make sure it is never part of the result")
347 unittest {
348     InjectIdBuilder builder;
349     builder.put(MutationStatusId(1), Checksum(1, 2));
350     builder.put(MutationStatusId(2), Checksum(3, 4));
351     builder.put(MutationStatusId(3), Checksum(1, 2));
352     auto r = builder.finalize;
353 
354     assert(r.front.statusId == MutationStatusId(2));
355     r.popFront;
356     assert(r.empty);
357 }