dextool_test.integration source code

1 /**
2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module dextool_test.integration;
7 
8 import scriptlike;
9 import unit_threaded : shouldEqual;
10 
11 import dextool_test;
12 
13 enum globalTestdir = "fuzzer_tests";
14 
15 struct Compile {
16     enum Kind {
17         main,
18         fuzzTest,
19         original,
20     }
21 
22     Kind kind;
23     Path payload;
24     alias payload this;
25 }
26 
27 struct TestParams {
28     bool doCompare = true;
29     bool doCompile = true;
30 
31     Path root;
32     Path testRoot;
33 
34     Path input;
35     Path input_impl;
36     Path[2][] output;
37 
38     Compile[] compileFiles;
39 
40     // dextool parameters;
41     string[] dexParams;
42     string[] dexFlags;
43 
44     // Compiler
45     string[] compileFlags;
46     string[] compileIncls;
47     string originalCompilerVar;
48 }
49 
50 TestParams genTestParams(string hdr, string impl, const ref TestEnv testEnv) {
51     TestParams p;
52 
53     p.root = Path("plugin_testdata").absolutePath;
54     p.input = p.root ~ Path(hdr);
55     p.input_impl = p.root ~ Path(impl);
56     p.testRoot = p.input.dirName;
57 
58     p.output ~= [p.input ~ Ext(".hpp.ref"), testEnv.outdir ~ "dextool_default_scheduler.hpp"];
59 
60     p.compileFiles ~= Compile(Compile.Kind.main, testEnv.outdir ~ "main.cpp");
61     p.compileFiles ~= Compile(Compile.Kind.original, p.input_impl);
62 
63     p.dexParams = ["fuzzer", "--debug"];
64     p.dexFlags = [];
65 
66     p.compileFlags = compilerFlags();
67     p.compileIncls = ["-I" ~ p.input.dirName.toString];
68 
69     if (p.input_impl.extension == ".c") {
70         p.originalCompilerVar = "CC";
71     } else {
72         p.originalCompilerVar = "CXX";
73     }
74 
75     return p;
76 }
77 
78 void runTestFile(ref TestParams p, ref TestEnv testEnv, Flag!"sortLines" sortLines = No.sortLines,
79         Flag!"skipComments" skipComments = Yes.skipComments) {
80     dextoolYap("Input:%s", p.input.raw);
81     runDextool(p.input, testEnv, p.dexParams, p.dexFlags);
82 
83     if (p.doCompare) {
84         dextoolYap("Comparing");
85         auto input = p.input.stripExtension;
86         foreach (o; p.output) {
87             compareResult(sortLines, skipComments, GR(o[0], o[1]));
88         }
89     }
90 
91     if (p.doCompile) {
92         dextoolYap("Compiling");
93 
94         // these only exist AFTER dextool has executed.
95         auto comp_files = p.compileFiles ~ dirEntries(testEnv.outdir, "dextool_fuzz_case*.cpp",
96                 SpanMode.shallow).map!(a => Compile(Compile.Kind.fuzzTest, Path(a))).array();
97         dextoolYap("Found generated fuzz tests: %s", comp_files);
98 
99         compile(testEnv.outdir ~ "binary", comp_files, p.compileFlags,
100                 p.compileIncls, p.originalCompilerVar, testEnv);
101     }
102 }
103 
104 void compile(const Path dst_binary, Compile[] files, const string[] flags,
105         const string[] incls, string original_var, const ref TestEnv testEnv) {
106     import core.sys.posix.sys.stat;
107     import std.file : setAttributes;
108 
109     immutable bool[string] rm_flag = ["-Wpedantic" : true, "-Werror" : true, "-pedantic" : true];
110 
111     auto flags_ = flags.filter!(a => a !in rm_flag).array();
112 
113     Args compile_args;
114     compile_args ~= "-g";
115     compile_args ~= "-I" ~ testEnv.outdir.escapePath;
116     compile_args ~= "-I" ~ "support";
117     compile_args ~= incls.dup;
118 
119     Args wrapper_args;
120     wrapper_args ~= flags_.dup;
121     wrapper_args ~= "-o$OUT";
122     wrapper_args ~= files.filter!(a => a.kind != Compile.Kind.original)
123         .map!(a => a.payload).array().dup;
124     wrapper_args ~= (testEnv.outdir ~ "original.o").toString;
125     wrapper_args ~= "-l" ~ "dextoolfuzz";
126     wrapper_args ~= "-L.";
127 
128     Args original_args;
129     original_args ~= "-c";
130     original_args ~= "-o" ~ (testEnv.outdir ~ "original.o").toString;
131     original_args ~= files.filter!(a => a.kind == Compile.Kind.original)
132         .map!(a => a.payload).array().dup;
133 
134     string script_p = (testEnv.outdir ~ "build.sh").toString;
135     auto script = File(script_p, "w");
136     script.writeln("#!/bin/bash");
137     script.writeln("if [[ -z $CC ]]; then CC=gcc; fi");
138     script.writeln("if [[ -z $CXX ]]; then CXX=g++; fi");
139     script.writeln("if [[ -z $OUT ]]; then OUT=" ~ dst_binary.escapePath ~ "; fi");
140     script.writefln("$%s $FLAGS %s %s", original_var, compile_args.data, original_args.data);
141     script.writefln("$CXX $WRAPFLAGS %s %s", compile_args.data, wrapper_args.data);
142     script.close();
143 
144     auto script_attr = getAttributes(script_p);
145     script_attr = script_attr | S_IRWXU;
146     setAttributes(script_p, script_attr);
147 
148     runAndLog(script_p).status.shouldEqual(0);
149 }
150 
151 @(testId ~ "shall be a fuzzer environment for a void function")
152 unittest {
153     mixin(envSetup(globalTestdir));
154 
155     auto p = genTestParams("stage_1/a_void_func.h", "stage_1/a_void_func.c", testEnv);
156     p.doCompare = false;
157     runTestFile(p, testEnv);
158 }
159 
160 @(testId ~ "shall be a fuzzer environment for a function with parameters of primitive data types")
161 unittest {
162     mixin(envSetup(globalTestdir));
163 
164     auto p = genTestParams("stage_1/func_with_primitive_params.h",
165             "stage_1/func_with_primitive_params.c", testEnv);
166     p.doCompare = false;
167     runTestFile(p, testEnv);
168 }
169 
170 @(testId ~ "shall be a fuzzer environment for a function with int params and lots of if-stmt")
171 unittest {
172     mixin(envSetup(globalTestdir));
173 
174     auto p = genTestParams("stage_2/int_param_nested_if.hpp",
175             "stage_2/int_param_nested_if.cpp", testEnv);
176     p.doCompare = false;
177     runTestFile(p, testEnv);
178 }
179 
180 @(testId ~ "shall be a fuzzer environment with limits from a config file")
181 unittest {
182     mixin(envSetup(globalTestdir));
183 
184     auto p = genTestParams("stage_2/limit_params.hpp", "stage_2/limit_params.cpp", testEnv);
185     p.dexParams ~= "--config=" ~ (p.root ~ "stage_2/limit_params.xml").toString;
186     p.doCompare = false;
187     runTestFile(p, testEnv);
188 }
189 
190 @(testId ~ "shall be a fuzzer environment with complex structs and transforms from a config file ")
191 unittest {
192     mixin(envSetup(globalTestdir));
193 
194     auto p = genTestParams("stage_2/transform_param.hpp", "stage_2/transform_param.cpp", testEnv);
195     p.dexParams ~= "--config=" ~ (p.root ~ "stage_2/transform_param.xml").toString;
196     runTestFile(p, testEnv);
197 }
198 
199 @(testId ~ "shall be a fuzzer environment fuzzing in a limited range following a config")
200 unittest {
201     mixin(envSetup(globalTestdir));
202 
203     auto p = genTestParams("stage_2/user_param_fuzzer.hpp",
204             "stage_2/user_param_fuzzer.cpp", testEnv);
205     p.dexParams ~= "--config=" ~ (p.root ~ "stage_2/user_param_fuzzer.xml").toString;
206     p.doCompare = false;
207     runTestFile(p, testEnv);
208 }
209 
210 @("shall be a working, complete test of the helper library")
211 unittest {
212     mixin(envSetup(globalTestdir));
213 
214     auto p = genTestParams("", "test_lib/dummy.cpp", testEnv);
215     // dfmt off
216     auto comp_files =
217         [Compile(Compile.Kind.main, p.root ~ Path("test_lib") ~ "main.cpp"),
218         Compile(Compile.Kind.original, p.root ~ Path("test_lib") ~ "dummy.cpp")]
219             ~
220         ["test_fuzz_helper.cpp"]
221         .map!(a => p.root ~ Path("test_lib") ~ a)
222         .map!(a => Compile(Compile.Kind.fuzzTest, a))
223         .array();
224     // dfmt on
225 
226     compile(testEnv.outdir ~ "binary", comp_files, p.compileFlags,
227             p.compileIncls, p.originalCompilerVar, testEnv);
228 
229     Args test;
230     test ~= (testEnv.outdir ~ "binary").toString;
231     test ~= (p.root ~ Path("test_lib") ~ "raw_dummy_data.cpp").toString;
232 
233     runAndLog(test).status.shouldEqual(0);
234 }