1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 Test a mutant by modifying the source code.
11 */
12 module dextool.plugin.mutate.backend.test_mutant.source_mutant;
13 
14 import core.time : Duration;
15 import logger = std.experimental.logger;
16 import std.algorithm : sort, map;
17 import std.array : empty, array;
18 import std.exception : collectException;
19 import std.path : buildPath;
20 
21 import my.fsm : next, act, get, TypeDataMap;
22 import my.named_type;
23 import proc : DrainElement;
24 import sumtype;
25 
26 static import my.fsm;
27 
28 import dextool.plugin.mutate.backend.database : Database, MutationEntry;
29 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
30 import dextool.plugin.mutate.backend.test_mutant.common;
31 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner;
32 import dextool.plugin.mutate.backend.type : Mutation, TestCase;
33 import dextool.plugin.mutate.config;
34 import dextool.plugin.mutate.type : ShellCommand;
35 import dextool.type : AbsolutePath, Path;
36 
37 @safe:
38 
39 /** Drive the control flow when testing **a** mutant.
40  */
41 struct MutationTestDriver {
42     import std.datetime.stopwatch : StopWatch;
43     import std.typecons : Tuple;
44 
45     static struct Global {
46         FilesysIO fio;
47         Database* db;
48 
49         /// The mutant to apply.
50         MutationEntry mutp;
51 
52         /// Runs the test commands.
53         TestRunner* runner;
54 
55         /// File to mutate.
56         AbsolutePath mutateFile;
57 
58         /// The original file.
59         Blob original;
60 
61         /// The result of running the test cases.
62         TestResult testResult;
63 
64         /// Test cases that killed the mutant.
65         TestCase[] testCases;
66 
67         /// How long it took to do the mutation testing.
68         StopWatch swCompile;
69         StopWatch swTest;
70     }
71 
72     static struct None {
73     }
74 
75     static struct Initialize {
76     }
77 
78     static struct MutateCode {
79         NamedType!(bool, Tag!"FilesysError", bool.init, TagStringable, ImplicitConvertable) filesysError;
80         NamedType!(bool, Tag!"MutationError", bool.init, TagStringable, ImplicitConvertable) mutationError;
81     }
82 
83     static struct TestMutantData {
84         /// If the user has configured that the test cases should be analyzed.
85         bool hasTestCaseOutputAnalyzer;
86         ShellCommand buildCmd;
87         Duration buildCmdTimeout;
88     }
89 
90     static struct TestMutant {
91         NamedType!(bool, Tag!"HasTestOutput", bool.init, TagStringable, ImplicitConvertable) hasTestOutput;
92     }
93 
94     static struct RestoreCode {
95         NamedType!(bool, Tag!"FilesysError", bool.init, TagStringable, ImplicitConvertable) filesysError;
96     }
97 
98     static struct TestCaseAnalyzeData {
99         TestCaseAnalyzer* testCaseAnalyzer;
100     }
101 
102     static struct TestCaseAnalyze {
103         bool unstableTests;
104     }
105 
106     static struct StoreResult {
107     }
108 
109     static struct Done {
110     }
111 
112     static struct FilesysError {
113     }
114 
115     // happens when an error occurs during mutations testing but that do not
116     // prohibit testing of other mutants
117     static struct NoResultRestoreCode {
118     }
119 
120     static struct NoResult {
121     }
122 
123     alias Fsm = my.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode,
124             TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult);
125     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData);
126 
127     private {
128         Fsm fsm;
129         Global global;
130         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local;
131         bool isRunning_ = true;
132         bool stopBecauseError_;
133     }
134 
135     MutationTestResult[] result;
136 
137     this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) {
138         this.global = global;
139         this.local = LocalStateDataT(l1, l2);
140 
141         if (logger.globalLogLevel == logger.LogLevel.trace)
142             fsm.logger = (string s) { logger.trace(s); };
143     }
144 
145     static void execute_(ref MutationTestDriver self) @trusted {
146         self.fsm.next!((None a) => fsm(Initialize.init),
147                 (Initialize a) => fsm(MutateCode.init), (MutateCode a) {
148             if (a.filesysError)
149                 return fsm(FilesysError.init);
150             else if (a.mutationError)
151                 return fsm(NoResultRestoreCode.init);
152             return fsm(TestMutant.init);
153         }, (TestMutant a) {
154             if (self.global.testResult.status == Mutation.Status.killed
155                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) {
156                 return fsm(TestCaseAnalyze.init);
157             }
158             return fsm(RestoreCode.init);
159         }, (TestCaseAnalyze a) {
160             if (a.unstableTests)
161                 return fsm(NoResultRestoreCode.init);
162             return fsm(RestoreCode.init);
163         }, (RestoreCode a) {
164             if (a.filesysError)
165                 return fsm(FilesysError.init);
166             return fsm(StoreResult.init);
167         }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a),
168                 (FilesysError a) => fsm(a),
169                 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),);
170 
171         self.fsm.act!self;
172     }
173 
174 nothrow:
175 
176     void execute() {
177         try {
178             execute_(this);
179         } catch (Exception e) {
180             logger.warning(e.msg).collectException;
181         }
182     }
183 
184     /// Returns: true as long as the driver is processing a mutant.
185     bool isRunning() {
186         return isRunning_;
187     }
188 
189     bool stopBecauseError() {
190         return stopBecauseError_;
191     }
192 
193     void opCall(None data) {
194     }
195 
196     void opCall(Initialize data) {
197         global.swCompile.start;
198     }
199 
200     void opCall(Done data) {
201         isRunning_ = false;
202     }
203 
204     void opCall(FilesysError data) {
205         logger.warning("Filesystem error").collectException;
206         isRunning_ = false;
207         stopBecauseError_ = true;
208     }
209 
210     void opCall(NoResultRestoreCode data) {
211         RestoreCode tmp;
212         this.opCall(tmp);
213     }
214 
215     void opCall(NoResult data) {
216         isRunning_ = false;
217     }
218 
219     void opCall(ref MutateCode data) {
220         import dextool.plugin.mutate.backend.generate_mutant : generateMutant,
221             GenerateMutantResult, GenerateMutantStatus;
222 
223         try {
224             global.mutateFile = AbsolutePath(buildPath(global.fio.getOutputDir, global.mutp.file));
225             global.original = global.fio.makeInput(global.mutateFile);
226         } catch (Exception e) {
227             logger.error(e.msg).collectException;
228             logger.warning("Unable to read ", global.mutateFile).collectException;
229             data.filesysError.get = true;
230             return;
231         }
232 
233         // mutate
234         try {
235             auto fout = global.fio.makeOutput(global.mutateFile);
236             auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout);
237 
238             final switch (mut_res.status) with (GenerateMutantStatus) {
239             case error:
240                 data.mutationError.get = true;
241                 break;
242             case filesysError:
243                 data.filesysError.get = true;
244                 break;
245             case databaseError:
246                 // such as when the database is locked
247                 data.mutationError.get = true;
248                 break;
249             case checksumError:
250                 data.filesysError.get = true;
251                 break;
252             case noMutation:
253                 data.mutationError.get = true;
254                 break;
255             case ok:
256                 try {
257                     logger.infof("from '%s' to '%s' in %s:%s:%s",
258                             cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to,
259                             global.mutateFile, global.mutp.sloc.line, global.mutp.sloc.column);
260                     logger.trace(global.mutp.id).collectException;
261                 } catch (Exception e) {
262                     logger.warningf("%s %s", global.mutp.id, e.msg);
263                 }
264                 break;
265             }
266         } catch (Exception e) {
267             logger.warning(e.msg).collectException;
268             data.mutationError.get = true;
269         }
270     }
271 
272     void opCall(ref TestMutant data) {
273         bool successCompile;
274         compile(local.get!TestMutant.buildCmd, local.get!TestMutant.buildCmdTimeout).match!(
275                 (Mutation.Status a) { global.testResult.status = a; }, (bool success) {
276             successCompile = success;
277         },);
278 
279         global.swCompile.stop;
280         global.swTest.start;
281 
282         if (!successCompile)
283             return;
284 
285         global.testResult = runTester(*global.runner);
286 
287         data.hasTestOutput.get = !global.testResult.output.empty;
288     }
289 
290     void opCall(ref TestCaseAnalyze data) {
291         scope (exit)
292             global.testResult.output = null;
293 
294         try {
295             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(
296                     global.testResult.output);
297 
298             analyze.match!((TestCaseAnalyzer.Success a) {
299                 global.testCases = a.failed;
300                 global.testCases ~= global.testResult.testCmds.map!(a => TestCase(a.toShortString))
301                     .array;
302             }, (TestCaseAnalyzer.Unstable a) {
303                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
304                 logger.info(
305                     "As configured the result is ignored which will force the mutant to be re-tested");
306                 data.unstableTests = true;
307             }, (TestCaseAnalyzer.Failed a) {
308                 logger.warning("The parser that analyze the output from test case(s) failed");
309             });
310         } catch (Exception e) {
311             logger.warning(e.msg).collectException;
312         }
313     }
314 
315     void opCall(StoreResult data) {
316         import miniorm : spinSql;
317 
318         const statusId = spinSql!(() => global.db.getMutationStatusId(global.mutp.id));
319 
320         global.swTest.stop;
321         auto profile = MutantTimeProfile(global.swCompile.peek, global.swTest.peek);
322 
323         if (statusId.isNull) {
324             logger.trace("No MutationStatusId for ", global.mutp.id.get).collectException;
325             return;
326         }
327 
328         result = [
329             MutationTestResult(global.mutp.id, statusId.get, global.testResult.status,
330                     profile, global.testCases, global.testResult.exitStatus)
331         ];
332 
333         logger.infof("%s:%s (%s)", global.testResult.status,
334                 global.testResult.exitStatus.get, profile).collectException;
335         logger.infof(!global.testCases.empty, `killed by [%-(%s, %)]`,
336                 global.testCases.sort.map!"a.name").collectException;
337     }
338 
339     void opCall(ref RestoreCode data) {
340         // restore the original file.
341         try {
342             global.fio.makeOutput(global.mutateFile).write(global.original.content);
343         } catch (Exception e) {
344             logger.error(e.msg).collectException;
345             // fatal error because being unable to restore a file prohibit
346             // future mutations.
347             data.filesysError.get = true;
348         }
349     }
350 }