1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 Test a mutant by modifying the source code.
11 */
12 module dextool.plugin.mutate.backend.test_mutant.source_mutant;
13 
14 import core.time : Duration;
15 import logger = std.experimental.logger;
16 import std.array : empty;
17 import std.exception : collectException;
18 import std.path : buildPath;
19 
20 import sumtype;
21 import proc : DrainElement;
22 import my.fsm : Fsm, next, act, get, TypeDataMap;
23 
24 static import my.fsm;
25 
26 import dextool.plugin.mutate.backend.database : Database, MutationEntry;
27 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
28 import dextool.plugin.mutate.backend.test_mutant.common;
29 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
30 import dextool.plugin.mutate.backend.type : Mutation, TestCase;
31 import dextool.plugin.mutate.config;
32 import dextool.plugin.mutate.type : ShellCommand;
33 import dextool.type : AbsolutePath, Path;
34 
35 @safe:
36 
37 /// The result of testing a mutant.
38 struct MutationTestResult {
39     import std.datetime : Duration;
40     import sumtype;
41     import proc : DrainElement;
42     import dextool.plugin.mutate.backend.database : MutationId;
43     import dextool.plugin.mutate.backend.type : TestCase;
44 
45     static struct NoResult {
46     }
47 
48     static struct StatusUpdate {
49         MutationId id;
50         Mutation.Status status;
51         Duration testTime;
52         TestCase[] testCases;
53     }
54 
55     alias Value = SumType!(NoResult, StatusUpdate);
56     Value value;
57 
58     void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc {
59         this.value = rhs.value;
60     }
61 
62     void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc {
63         this.value = Value(rhs);
64     }
65 }
66 
67 /** Drive the control flow when testing **a** mutant.
68  */
69 struct MutationTestDriver {
70     import std.datetime.stopwatch : StopWatch;
71     import std.typecons : Tuple;
72     import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase;
73 
74     static struct Global {
75         FilesysIO fio;
76         Database* db;
77 
78         /// The mutant to apply.
79         MutationEntry mutp;
80 
81         /// Runs the test commands.
82         TestRunner* runner;
83 
84         /// File to mutate.
85         AbsolutePath mut_file;
86 
87         /// The original file.
88         Blob original;
89 
90         /// The result of running the test cases.
91         Mutation.Status mut_status;
92 
93         /// Test cases that killed the mutant.
94         TestCase[] test_cases;
95 
96         /// How long it took to do the mutation testing.
97         StopWatch sw;
98     }
99 
100     static struct None {
101     }
102 
103     static struct Initialize {
104     }
105 
106     static struct MutateCode {
107         bool next;
108         bool filesysError;
109         bool mutationError;
110     }
111 
112     static struct TestMutantData {
113         /// If the user has configured that the test cases should be analyzed.
114         bool hasTestCaseOutputAnalyzer;
115         ShellCommand buildCmd;
116         Duration buildCmdTimeout;
117     }
118 
119     static struct TestMutant {
120         bool hasTestOutput;
121     }
122 
123     static struct RestoreCode {
124         bool next;
125         bool filesysError;
126     }
127 
128     static struct TestCaseAnalyzeData {
129         TestCaseAnalyzer* testCaseAnalyzer;
130         DrainElement[] output;
131     }
132 
133     static struct TestCaseAnalyze {
134         bool unstableTests;
135     }
136 
137     static struct StoreResult {
138     }
139 
140     static struct Done {
141     }
142 
143     static struct FilesysError {
144     }
145 
146     // happens when an error occurs during mutations testing but that do not
147     // prohibit testing of other mutants
148     static struct NoResultRestoreCode {
149     }
150 
151     static struct NoResult {
152     }
153 
154     alias Fsm = my.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode,
155             TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult);
156     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData);
157 
158     private {
159         Fsm fsm;
160         Global global;
161         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local;
162         bool isRunning_ = true;
163         bool stopBecauseError_;
164     }
165 
166     MutationTestResult result;
167 
168     this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) {
169         this.global = global;
170         this.local = LocalStateDataT(l1, l2);
171     }
172 
173     static void execute_(ref MutationTestDriver self) @trusted {
174         self.fsm.next!((None a) => fsm(Initialize.init),
175                 (Initialize a) => fsm(MutateCode.init), (MutateCode a) {
176             if (a.next)
177                 return fsm(TestMutant.init);
178             else if (a.filesysError)
179                 return fsm(FilesysError.init);
180             else if (a.mutationError)
181                 return fsm(NoResultRestoreCode.init);
182             return fsm(a);
183         }, (TestMutant a) {
184             if (self.global.mut_status == Mutation.Status.killed
185                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) {
186                 return fsm(TestCaseAnalyze.init);
187             }
188             return fsm(RestoreCode.init);
189         }, (TestCaseAnalyze a) {
190             if (a.unstableTests)
191                 return fsm(NoResultRestoreCode.init);
192             return fsm(RestoreCode.init);
193         }, (RestoreCode a) {
194             if (a.next)
195                 return fsm(StoreResult.init);
196             else if (a.filesysError)
197                 return fsm(FilesysError.init);
198             return fsm(a);
199         }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a),
200                 (FilesysError a) => fsm(a),
201                 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),);
202 
203         debug logger.trace("state: ", self.fsm.logNext);
204         self.fsm.act!(self);
205     }
206 
207 nothrow:
208 
209     void execute() {
210         try {
211             execute_(this);
212         } catch (Exception e) {
213             logger.warning(e.msg).collectException;
214         }
215     }
216 
217     /// Returns: true as long as the driver is processing a mutant.
218     bool isRunning() {
219         return isRunning_;
220     }
221 
222     bool stopBecauseError() {
223         return stopBecauseError_;
224     }
225 
226     void opCall(None data) {
227     }
228 
229     void opCall(Initialize data) {
230         global.sw.start;
231     }
232 
233     void opCall(Done data) {
234         isRunning_ = false;
235     }
236 
237     void opCall(FilesysError data) {
238         logger.warning("Filesystem error").collectException;
239         isRunning_ = false;
240         stopBecauseError_ = true;
241     }
242 
243     void opCall(NoResultRestoreCode data) {
244         RestoreCode tmp;
245         this.opCall(tmp);
246     }
247 
248     void opCall(NoResult data) {
249         isRunning_ = false;
250     }
251 
252     void opCall(ref MutateCode data) {
253         import dextool.plugin.mutate.backend.generate_mutant : generateMutant,
254             GenerateMutantResult, GenerateMutantStatus;
255 
256         try {
257             global.mut_file = AbsolutePath(buildPath(global.fio.getOutputDir, global.mutp.file));
258             global.original = global.fio.makeInput(global.mut_file);
259         } catch (Exception e) {
260             logger.error(e.msg).collectException;
261             logger.warning("Unable to read ", global.mut_file).collectException;
262             data.filesysError = true;
263             return;
264         }
265 
266         // mutate
267         try {
268             auto fout = global.fio.makeOutput(global.mut_file);
269             auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout);
270 
271             final switch (mut_res.status) with (GenerateMutantStatus) {
272             case error:
273                 data.mutationError = true;
274                 break;
275             case filesysError:
276                 data.filesysError = true;
277                 break;
278             case databaseError:
279                 // such as when the database is locked
280                 data.mutationError = true;
281                 break;
282             case checksumError:
283                 data.filesysError = true;
284                 break;
285             case noMutation:
286                 data.mutationError = true;
287                 break;
288             case ok:
289                 data.next = true;
290                 try {
291                     logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id,
292                             cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to,
293                             global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column);
294 
295                 } catch (Exception e) {
296                     logger.warning("Mutation ID", e.msg);
297                 }
298                 break;
299             }
300         } catch (Exception e) {
301             logger.warning(e.msg).collectException;
302             data.mutationError = true;
303         }
304     }
305 
306     void opCall(ref TestMutant data) {
307         global.mut_status = Mutation.Status.unknown;
308 
309         bool successCompile;
310         compile(local.get!TestMutant.buildCmd, local.get!TestMutant.buildCmdTimeout).match!(
311                 (Mutation.Status a) { global.mut_status = a; }, (bool success) {
312             successCompile = success;
313         },);
314 
315         if (!successCompile)
316             return;
317 
318         auto res = runTester(*global.runner);
319         global.mut_status = res.status;
320         data.hasTestOutput = !res.output.empty;
321         local.get!TestCaseAnalyze.output = res.output;
322     }
323 
324     void opCall(ref TestCaseAnalyze data) {
325         global.test_cases = null;
326 
327         try {
328             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(
329                     local.get!TestCaseAnalyze.output);
330             local.get!TestCaseAnalyze.output = null;
331 
332             analyze.match!((TestCaseAnalyzer.Success a) {
333                 global.test_cases = a.failed;
334             }, (TestCaseAnalyzer.Unstable a) {
335                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
336                 logger.info(
337                     "As configured the result is ignored which will force the mutant to be re-tested");
338                 data.unstableTests = true;
339             }, (TestCaseAnalyzer.Failed a) {
340                 logger.warning("The parser that analyze the output from test case(s) failed");
341             });
342         } catch (Exception e) {
343             logger.warning(e.msg).collectException;
344         }
345     }
346 
347     void opCall(StoreResult data) {
348         global.sw.stop;
349         result = MutationTestResult.StatusUpdate(global.mutp.id,
350                 global.mut_status, global.sw.peek, global.test_cases);
351     }
352 
353     void opCall(ref RestoreCode data) {
354         // restore the original file.
355         try {
356             global.fio.makeOutput(global.mut_file).write(global.original.content);
357         } catch (Exception e) {
358             logger.error(e.msg).collectException;
359             // fatal error because being unable to restore a file prohibit
360             // future mutations.
361             data.filesysError = true;
362             return;
363         }
364 
365         data.next = true;
366     }
367 }