1 /**
2 Copyright: Copyright (c) 2018, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.config;
11 
12 import core.time : Duration, dur;
13 import std.typecons : Nullable;
14 
15 import my.filter : GlobFilter;
16 import my.named_type;
17 import my.optional;
18 
19 import dextool.plugin.mutate.type;
20 import dextool.type : AbsolutePath, Path;
21 public import dextool.plugin.mutate.backend.type : Mutation, TestGroup;
22 
23 /// The mode the tool is operating in
24 enum ToolMode {
25     /// No mode set
26     none,
27     /// analyze for mutation points
28     analyzer,
29     /// center that can operate and control subcomponents
30     generate_mutant,
31     /// test mutation points with a test suite
32     test_mutants,
33     /// generate a report of the mutation points
34     report,
35     /// administrator interface for the mutation database
36     admin,
37     /// Dump the TOML configuration to the console
38     dumpConfig,
39     /// Write a TOML config to the filesystem
40     initConfig,
41 }
42 
43 /// Config of the report.
44 struct ConfigReport {
45     ReportKind reportKind;
46     ReportSection[] reportSection = [ReportSection.summary];
47 
48     /// Directory to write logs to when writing to the filesystem.
49     AbsolutePath logDir;
50 
51     /// Controls how to sort test cases by their kill statistics.
52     ReportKillSortOrder tcKillSortOrder;
53     int tcKillSortNum = 20;
54 
55     /// User regex for reporting groups of tests
56     TestGroup[] testGroups;
57 
58     /// If a unified diff should be used in the report
59     bool unifiedDiff;
60 
61     /// If profiling data should be printed.
62     bool profile;
63 
64     NamedType!(uint, Tag!"HighInterestMutantsNr", uint.init, TagStringable) highInterestMutantsNr = 5;
65 
66     alias TestMetaData = NamedType!(AbsolutePath, Tag!"TestMetaData",
67             AbsolutePath.init, TagStringable);
68     Optional!TestMetaData testMetadata;
69 }
70 
71 /// Configuration data for the compile_commands.json
72 struct ConfigCompileDb {
73     import dextool.compilation_db : CompileCommandFilter;
74 
75     /// Raw user input via either config or cli
76     string[] rawDbs;
77 
78     /// path to compilation databases.
79     AbsolutePath[] dbs;
80 
81     /// Flags the user wants to be automatically removed from the compile_commands.json.
82     CompileCommandFilter flagFilter;
83 }
84 
85 /// Configuration of how the mutation analyzer should act.
86 struct ConfigAnalyze {
87     /// Mutants to analyze for and save in the database.
88     MutationKind[] mutation;
89 
90     /// User input of excludes before they are adjusted to relative root
91     string[] rawExclude;
92     /// User input of includes before they are adjusted to relative root
93     string[] rawInclude;
94 
95     /// The constructed glob filter which based on rawExclude and rawinclude.
96     GlobFilter fileMatcher;
97 
98     /// The size of the thread pool which affects how many files are analyzed in parallel.
99     int poolSize;
100 
101     /// What files to analyze is derived from a diff.
102     bool unifiedDiffFromStdin;
103 
104     /// Remove files from the database that aren't found when analyzing.
105     bool prune;
106 
107     /// Turn off the sqlite synchronization safety
108     bool fastDbStore;
109 
110     /// If profiling data should be printed.
111     bool profile;
112 
113     /// Force the result from the files to always be saved
114     bool forceSaveAnalyze;
115 
116     /// User file/directories containing tests to checksum and timestamp
117     string[] rawTestPaths;
118     AbsolutePath[] testPaths;
119 
120     /// User input of excludes before they are adjusted to relative root
121     string[] rawTestExclude;
122     /// User input of includes before they are adjusted to relative root
123     string[] rawTestInclude;
124 
125     /// The constructed glob filter which based on rawExclude and rawinclude.
126     GlobFilter testFileMatcher;
127 
128     /// Which mutation ID generator to use.
129     MutantIdGeneratorConfig idGenConfig;
130 }
131 
132 /// Settings for the compiler
133 struct ConfigCompiler {
134     import dextool.compilation_db : SystemCompiler = Compiler;
135 
136     /// Additional flags the user wants to add besides those that are in the compile_commands.json.
137     string[] extraFlags;
138 
139     /// True requires system includes to be passed on to the compiler via -I
140     bool forceSystemIncludes;
141 
142     /// Deduce compiler flags from this compiler and not the one in the
143     /// supplied compilation database.  / This is needed when the one specified
144     /// in the DB has e.g. a c++ stdlib that is not compatible with clang.
145     SystemCompiler useCompilerSystemIncludes;
146 
147     NamedType!(bool, Tag!"AllowErrors", bool.init, TagStringable) allowErrors;
148 }
149 
150 /// Settings for mutation testing
151 struct ConfigMutationTest {
152     ShellCommand[] mutationTester;
153 
154     enum TestCmdDirSearch {
155         recursive,
156         shallow,
157     }
158 
159     /// Find executables in this directory and add them to mutationTester.
160     Path[] testCommandDir;
161 
162     /// Flags to add to all executables found in `testCommandDir`
163     string[] testCommandDirFlag;
164 
165     TestCmdDirSearch testCmdDirSearch;
166 
167     ShellCommand mutationCompile;
168     ShellCommand[] mutationTestCaseAnalyze;
169     TestCaseAnalyzeBuiltin[] mutationTestCaseBuiltin;
170 
171     /// If the user hard code a timeout for the test suite.
172     Nullable!Duration mutationTesterRuntime;
173 
174     string metadataPath;
175 
176     /// Timeout to use when compiling.
177     Duration buildCmdTimeout = 30.dur!"minutes";
178 
179     /// In what order to choose mutants to test.
180     MutationOrder mutationOrder = MutationOrder.bySize;
181     bool dryRun;
182 
183     /// How to behave when new test cases are detected.
184     enum NewTestCases {
185         doNothing,
186         /// Automatically reset alive mutants
187         resetAlive,
188     }
189 
190     NewTestCases onNewTestCases;
191 
192     /// How to behave when test cases are detected of having been removed
193     enum RemovedTestCases {
194         doNothing,
195         /// Remove it and all results connectedto the test case
196         remove,
197     }
198 
199     RemovedTestCases onRemovedTestCases;
200 
201     /// How to behave when mutants have aged.
202     enum OldMutant {
203         nothing,
204         test,
205     }
206 
207     OldMutant onOldMutants;
208     long oldMutantsNr;
209     NamedType!(double, Tag!"OldMutantPercentage", double.init, TagStringable) oldMutantPercentage = 0.0;
210 
211     /// Max time to run mutation testing.
212     // note that Duration.max + Clock.currTime results in a negative time...
213     Duration maxRuntime = 52.dur!"weeks";
214 
215     // Constrain the mutation testing.
216     TestConstraint constraint;
217 
218     /// If constraints should be read from a unified diff via stdin.
219     bool unifiedDiffFromStdin;
220 
221     /// Stop after this many alive mutants are found. Only effective if constraint.empty is false.
222     Nullable!int maxAlive;
223 
224     /// The size of the thread pool which affects how many tests are executed in parallel.
225     int testPoolSize;
226 
227     /// If early stopping of test command execution should be used
228     bool useEarlyTestCmdStop;
229 
230     enum LoadBehavior {
231         nothing,
232         /// Slow the testing until the load goes below the threshold
233         slowdown,
234         /// Stop mutation testing if the 15min load average reach this number.
235         halt,
236     }
237 
238     LoadBehavior loadBehavior;
239     NamedType!(double, Tag!"LoadThreshold", double.init, TagStringable) loadThreshold;
240 
241     /// Continuesly run the test suite to see that the test suite is OK when no mutants are injected.
242     NamedType!(bool, Tag!"ContinuesCheckTestSuite", bool.init, TagStringable) contCheckTestSuite;
243     NamedType!(int, Tag!"ContinuesCheckTestSuitePeriod", int.init, TagStringable) contCheckTestSuitePeriod = 100;
244 
245     NamedType!(bool, Tag!"TestCmdChecksum", bool.init, TagStringable) testCmdChecksum;
246 
247     NamedType!(long, Tag!"MaxTestCaseOutputCaptureMbyte", int.init, TagStringable) maxTestCaseOutput = 10;
248 
249     NamedType!(bool, Tag!"UseSkipMutant", bool.init, TagStringable) useSkipMutant;
250 
251     NamedType!(double, Tag!"MaxMemoryUsage", double.init, TagStringable) maxMemUsage = 90.0;
252 }
253 
254 /// Settings for the administration mode
255 struct ConfigAdmin {
256     AdminOperation adminOp;
257     Mutation.Status mutantStatus;
258     Mutation.Status mutantToStatus;
259     string testCaseRegex;
260     long mutationStatusId;
261     string mutantRationale;
262 
263     /// used to specify a kind of mutation
264     Mutation.Kind[] subKind;
265 
266     /// Sub-mutation operators to run operation on.
267     MutationKind[] mutation;
268 }
269 
270 struct ConfigWorkArea {
271     /// User input root.
272     string rawRoot;
273 
274     AbsolutePath root;
275 
276     /// User input of excludes before they are adjusted to relative root
277     string[] rawExclude;
278     /// User input of includes before they are adjusted to relative root
279     string[] rawInclude;
280 
281     /// The constructed glob filter which based on rawExclude and rawinclude.
282     /// Only mutants whose location match will be generated.
283     GlobFilter mutantMatcher;
284 }
285 
286 /// Configuration of the generate mode.
287 struct ConfigGenerate {
288     long mutationStatusId;
289     MutationKind[] mutation;
290 }
291 
292 struct ConfigSchema {
293     bool use;
294 
295     SchemaRuntime runtime;
296 
297     /// Number of mutants to at most put in a schema (soft limit)
298     NamedType!(long, Tag!"MutantsPerSchema", long.init, TagStringable) mutantsPerSchema = 1000;
299 
300     /// Minimum number of mutants per schema for the schema to be saved in the database.
301     NamedType!(long, Tag!"MinMutantsPerSchema", long.init, TagStringable) minMutantsPerSchema = 3;
302 
303     /// Sanity check a schemata before it is used.
304     bool sanityCheckSchemata;
305 
306     /// If the schematas should be written to a separate file for offline inspection.
307     /// Write the instrumented source code to .cov.<ext> for separate inspection.
308     bool log;
309 
310     /// Stop mutation testing after the last schemata has been executed
311     bool stopAfterLastSchema;
312 
313     /// allows a user to control exactly which files the coverage and schemata
314     /// runtime is injected in.
315     UserRuntime[] userRuntimeCtrl;
316 
317     /// Only compile and execute the test suite. Used to train the schema generator.
318     NamedType!(bool, Tag!"SchemaTrainGenerator", bool.init, TagStringable) onlyCompile;
319 
320     /// Number of schema mutants to test in parallel.
321     int parallelMutants;
322 
323     /// The value which the timeout time is multiplied with
324     double timeoutScaleFactor = 2.0;
325 }
326 
327 struct ConfigCoverage {
328     bool use;
329 
330     CoverageRuntime runtime;
331 
332     /// If the generated coverage files should be saved.
333     bool log;
334 
335     /// allows a user to control exactly which files the coverage and schemata
336     /// runtime is injected in.
337     UserRuntime[] userRuntimeCtrl;
338 }