1 /**
2 Copyright: Copyright (c) 2015-2017, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.ctestdouble.frontend.ctestdouble;
11 
12 import std.typecons : Nullable;
13 
14 import logger = std.experimental.logger;
15 
16 import dextool.compilation_db;
17 import dextool.type;
18 
19 import dextool.plugin.types;
20 import dextool.plugin.ctestdouble.backend.cvariant : Controller, Parameters, Products;
21 import dextool.plugin.ctestdouble.frontend.types;
22 import dextool.plugin.ctestdouble.frontend.xml;
23 
24 // workaround for ldc-1.1.0 and dmd-2.071.2
25 auto workaround_linker_error() {
26     import cpptooling.testdouble.header_filter : TestDoubleIncludes,
27         GenericTestDoubleIncludes, DummyPayload;
28 
29     return typeid(GenericTestDoubleIncludes!DummyPayload).toString();
30 }
31 
32 struct RawConfiguration {
33     Nullable!XmlConfig xmlConfig;
34 
35     string[] fileExclude;
36     string[] fileRestrict;
37     string[] testDoubleInclude;
38     string[] inFiles;
39     string[] cflags;
40     string[] compileDb;
41     string header;
42     string headerFile;
43     string mainName = "TestDouble";
44     string mainFileName = "test_double";
45     string prefix = "Test_";
46     string stripInclude;
47     string out_;
48     string config;
49     bool help;
50     bool shortPluginHelp;
51     bool gmock;
52     bool generatePreInclude;
53     bool genPostInclude;
54     bool locationAsComment;
55     bool generateZeroGlobals;
56     bool invalidXmlConfig;
57 
58     string[] originalFlags;
59 
60     void parse(string[] args) {
61         import std.getopt;
62 
63         originalFlags = args.dup;
64 
65         try {
66             bool no_zero_globals;
67             // dfmt off
68             getopt(args, std.getopt.config.keepEndOfOptions, "h|help", &help,
69                    "short-plugin-help", &shortPluginHelp,
70                    "main", &mainName,
71                    "main-fname", &mainFileName,
72                    "out", &out_,
73                    "compile-db", &compileDb,
74                    "no-zeroglobals", &no_zero_globals,
75                    "prefix", &prefix,
76                    "strip-incl", &stripInclude,
77                    "header", &header,
78                    "header-file", &headerFile,
79                    "gmock", &gmock,
80                    "gen-pre-incl", &generatePreInclude,
81                    "gen-post-incl", &genPostInclude,
82                    "loc-as-comment", &locationAsComment,
83                    "td-include", &testDoubleInclude,
84                    "file-exclude", &fileExclude,
85                    "file-restrict", &fileRestrict,
86                    "in", &inFiles,
87                    "config", &config);
88             // dfmt on
89             generateZeroGlobals = !no_zero_globals;
90         } catch (std.getopt.GetOptException ex) {
91             logger.error(ex.msg);
92             help = true;
93         }
94 
95         // default arguments
96         if (stripInclude.length == 0) {
97             stripInclude = r".*/(.*)";
98             logger.trace("--strip-incl: using default regex to strip include path (basename)");
99         }
100 
101         if (config.length != 0) {
102             xmlConfig = readRawConfig(FileName(config));
103             if (xmlConfig.isNull) {
104                 invalidXmlConfig = true;
105             }
106         }
107 
108         import std.algorithm : find;
109         import std.array : array;
110         import std.range : drop;
111 
112         // at this point args contain "what is left". What is interesting then is those after "--".
113         cflags = args.find("--").drop(1).array();
114     }
115 
116     void printHelp() {
117         import std.stdio : writefln;
118 
119         writefln("%s\n\n%s\n%s", ctestdouble_opt.usage,
120                 ctestdouble_opt.optional, ctestdouble_opt.others);
121     }
122 
123     void dump() {
124         // TODO remove this
125         logger.tracef("args:
126 --header            :%s
127 --header-file       :%s
128 --file-restrict     :%s
129 --prefix            :%s
130 --gmock             :%s
131 --out               :%s
132 --file-exclude      :%s
133 --main              :%s
134 --strip-incl        :%s
135 --main-fname        :%s
136 --in                :%s
137 --compile-db        :%s
138 --gen-post-incl     :%s
139 --gen-pre-incl      :%s
140 --help              :%s
141 --loc-as-comment    :%s
142 --td-include        :%s
143 --no-zeroglobals    :%s
144 --config            :%s
145 CFLAGS              :%s
146 
147 xmlConfig           :%s", header, headerFile, fileRestrict, prefix, gmock,
148                 out_, fileExclude, mainName, stripInclude,
149                 mainFileName, inFiles, compileDb, genPostInclude, generatePreInclude, help, locationAsComment,
150                 testDoubleInclude, !generateZeroGlobals, config, cflags, xmlConfig);
151     }
152 }
153 
154 // dfmt off
155 static auto ctestdouble_opt = CliOptionParts(
156     "usage:
157  dextool ctestdouble [options] [--in=] [-- CFLAGS]",
158     // -------------
159     " --main=name        Used as part of interface, namespace etc [default: TestDouble]
160  --main-fname=n     Used as part of filename for generated files [default: test_double]
161  --prefix=p         Prefix used when generating test artifacts [default: Test_]
162  --strip-incl=r     A regexp used to strip the include paths
163  --gmock            Generate a gmock implementation of test double interface
164  --gen-pre-incl     Generate a pre include header file if it doesn't exist and use it
165  --gen-post-incl    Generate a post include header file if it doesn't exist and use it
166  --loc-as-comment   Generate a comment containing the location the symbol was derived from.
167                     Makes it easier to correctly define excludes/restricts
168  --header=s         Prepend generated files with the string
169  --header-file=f    Prepend generated files with the header read from the file
170  --no-zeroglobals   Turn off generation of the default implementation that zeroes globals
171  --config=path      Use configuration file",
172     // -------------
173 "others:
174  --in=              Input file to parse
175  --out=dir          directory for generated files [default: ./]
176  --compile-db=      Retrieve compilation parameters from the file
177  --file-exclude=    Exclude files from generation matching the regex
178  --file-restrict=   Restrict the scope of the test double to those files
179                     matching the regex
180  --td-include=      User supplied includes used instead of those found
181 
182 REGEX
183 The regex syntax is found at http://dlang.org/phobos/std_regex.html
184 
185 Information about --strip-incl.
186   Default regexp is: .*/(.*)
187 
188   To allow the user to selectively extract parts of the include path dextool
189   applies the regex and then concatenates all the matcher groups found.  It is
190   turned into the replacement include path.
191 
192   Important to remember then is that this approach requires that at least one
193   matcher group exists.
194 
195 Information about --file-exclude.
196   The regex must fully match the filename the AST node is located in.
197   If it matches all data from the file is excluded from the generated code.
198 
199 Information about --file-restrict.
200   The regex must fully match the filename the AST node is located in.
201   Only symbols from files matching the restrict affect the generated test double.
202 
203 EXAMPLES
204 
205 Generate a simple C test double.
206   dextool ctestdouble functions.h
207 
208   Analyze and generate a test double for function prototypes and extern variables.
209   Both those found in functions.h and outside, aka via includes.
210 
211   The test double is written to ./test_double.hpp/.cpp.
212   The name of the interface is Test_Double.
213 
214 Generate a C test double excluding data from specified files.
215   dextool ctestdouble --file-exclude=/foo.h --file-exclude='functions.[h,c]' --out=outdata/ functions.h -- -DBAR -I/some/path
216 
217   The code analyzer (Clang) will be passed the compiler flags -DBAR and -I/some/path.
218   During generation declarations found in foo.h or functions.h will be excluded.
219 
220   The file holding the test double is written to directory outdata.
221 "
222 );
223 // dfmt on
224 
225 struct FileData {
226     import dextool.type : FileName, WriteStrategy;
227 
228     FileName filename;
229     string data;
230     WriteStrategy strategy;
231 }
232 
233 /** Test double generation of C code.
234  *
235  * TODO Describe the options.
236  */
237 class CTestDoubleVariant : Controller, Parameters, Products {
238     import std.regex : regex, Regex;
239     import std.typecons : Flag;
240     import dextool.compilation_db : CompileCommandFilter;
241     import dextool.type : StubPrefix, FileName, DirName;
242     import cpptooling.testdouble.header_filter : TestDoubleIncludes, LocationType;
243     import dsrcgen.cpp : CppModule, CppHModule;
244 
245     private {
246         static const hdrExt = ".hpp";
247         static const implExt = ".cpp";
248         static const xmlExt = ".xml";
249 
250         StubPrefix prefix;
251 
252         DirName output_dir;
253         FileName main_file_hdr;
254         FileName main_file_impl;
255         FileName main_file_globals;
256         FileName gmock_file;
257         FileName pre_incl_file;
258         FileName post_incl_file;
259         FileName config_file;
260         FileName log_file;
261         CustomHeader custom_hdr;
262 
263         MainName main_name;
264         MainNs main_ns;
265         MainInterface main_if;
266         Flag!"Gmock" gmock;
267         Flag!"PreInclude" pre_incl;
268         Flag!"PostInclude" post_incl;
269         Flag!"locationAsComment" loc_as_comment;
270         Flag!"generateZeroGlobals" generate_zero_globals;
271 
272         Nullable!XmlConfig xmlConfig;
273         CompileCommandFilter compiler_flag_filter;
274         FilterSymbol restrict_symbols;
275         FilterSymbol exclude_symbols;
276 
277         Regex!char[] exclude;
278         Regex!char[] restrict;
279 
280         /// Data produced by the generatore intented to be written to specified file.
281         FileData[] file_data;
282 
283         TestDoubleIncludes td_includes;
284     }
285 
286     static auto makeVariant(ref RawConfiguration args) {
287         // dfmt off
288         auto variant = new CTestDoubleVariant(
289                 MainFileName(args.mainFileName), DirName(args.out_),
290                 regex(args.stripInclude))
291             .argPrefix(args.prefix)
292             .argMainName(args.mainName)
293             .argLocationAsComment(args.locationAsComment)
294             .argGmock(args.gmock)
295             .argPreInclude(args.generatePreInclude)
296             .argPostInclude(args.genPostInclude)
297             .argForceTestDoubleIncludes(args.testDoubleInclude)
298             .argFileExclude(args.fileExclude)
299             .argFileRestrict(args.fileRestrict)
300             .argCustomHeader(args.header, args.headerFile)
301             .argGenerateZeroGlobals(args.generateZeroGlobals)
302             .argXmlConfig(args.xmlConfig);
303         // dfmt on
304 
305         return variant;
306     }
307 
308     /** Design of c'tor.
309      *
310      * The c'tor has as paramters all the required configuration data.
311      * Assignment of members are used for optional configuration.
312      *
313      * Follows the design pattern "correct by construction".
314      *
315      * TODO document the parameters.
316      */
317     this(MainFileName main_fname, DirName output_dir, Regex!char strip_incl) {
318         this.output_dir = output_dir;
319         this.td_includes = TestDoubleIncludes(strip_incl);
320 
321         import std.path : baseName, buildPath, stripExtension;
322 
323         string base_filename = cast(string) main_fname;
324 
325         this.main_file_hdr = FileName(buildPath(cast(string) output_dir, base_filename ~ hdrExt));
326         this.main_file_impl = FileName(buildPath(cast(string) output_dir, base_filename ~ implExt));
327         this.main_file_globals = FileName(buildPath(cast(string) output_dir,
328                 base_filename ~ "_global" ~ implExt));
329         this.gmock_file = FileName(buildPath(cast(string) output_dir,
330                 base_filename ~ "_gmock" ~ hdrExt));
331         this.pre_incl_file = FileName(buildPath(cast(string) output_dir,
332                 base_filename ~ "_pre_includes" ~ hdrExt));
333         this.post_incl_file = FileName(buildPath(cast(string) output_dir,
334                 base_filename ~ "_post_includes" ~ hdrExt));
335         this.config_file = FileName(buildPath(output_dir, base_filename ~ "_config" ~ xmlExt));
336         this.log_file = FileName(buildPath(output_dir, base_filename ~ "_log" ~ xmlExt));
337     }
338 
339     auto argFileExclude(string[] a) {
340         import std.array : array;
341         import std.algorithm : map;
342 
343         this.exclude = a.map!(a => regex(a)).array();
344         return this;
345     }
346 
347     auto argFileRestrict(string[] a) {
348         import std.array : array;
349         import std.algorithm : map;
350 
351         this.restrict = a.map!(a => regex(a)).array();
352         return this;
353     }
354 
355     auto argPrefix(string s) {
356         this.prefix = StubPrefix(s);
357         return this;
358     }
359 
360     auto argMainName(string s) {
361         this.main_name = MainName(s);
362         this.main_ns = MainNs(s);
363         this.main_if = MainInterface("I_" ~ s);
364         return this;
365     }
366 
367     /// Force the includes to be those supplied by the user.
368     auto argForceTestDoubleIncludes(string[] a) {
369         if (a.length != 0) {
370             td_includes.forceIncludes(a);
371         }
372         return this;
373     }
374 
375     auto argCustomHeader(string header, string header_file) {
376         if (header.length != 0) {
377             this.custom_hdr = CustomHeader(header);
378         } else if (header_file.length != 0) {
379             import std.file : readText;
380 
381             string content = readText(header_file);
382             this.custom_hdr = CustomHeader(content);
383         }
384 
385         return this;
386     }
387 
388     auto argGmock(bool a) {
389         this.gmock = cast(Flag!"Gmock") a;
390         return this;
391     }
392 
393     auto argPreInclude(bool a) {
394         this.pre_incl = cast(Flag!"PreInclude") a;
395         return this;
396     }
397 
398     auto argPostInclude(bool a) {
399         this.post_incl = cast(Flag!"PostInclude") a;
400         return this;
401     }
402 
403     auto argLocationAsComment(bool a) {
404         this.loc_as_comment = cast(Flag!"locationAsComment") a;
405         return this;
406     }
407 
408     auto argGenerateZeroGlobals(bool value) {
409         this.generate_zero_globals = cast(Flag!"generateZeroGlobals") value;
410         return this;
411     }
412 
413     /** Ensure that the relevant information from the xml file is extracted.
414      *
415      * May overwrite information from the command line.
416      * TODO or should the command line have priority over the xml file?
417      */
418     auto argXmlConfig(Nullable!XmlConfig conf) {
419         import dextool.compilation_db : defaultCompilerFlagFilter;
420 
421         if (conf.isNull) {
422             compiler_flag_filter = CompileCommandFilter(defaultCompilerFlagFilter, 1);
423             return this;
424         }
425 
426         xmlConfig = conf;
427         compiler_flag_filter = CompileCommandFilter(conf.get.filterClangFlags,
428                 conf.get.skipCompilerArgs);
429         restrict_symbols = conf.get.restrictSymbols;
430         exclude_symbols = conf.get.excludeSymbols;
431 
432         return this;
433     }
434 
435     void processIncludes() {
436         td_includes.process();
437     }
438 
439     void finalizeIncludes() {
440         td_includes.finalize();
441     }
442 
443     /// Destination of the configuration file containing how the test double was generated.
444     FileName getXmlConfigFile() {
445         return config_file;
446     }
447 
448     /** Destination of the xml log for how dextool was ran when generatinng the
449      * test double.
450      */
451     FileName getXmlLog() {
452         return log_file;
453     }
454 
455     ref FilterSymbol getRestrictSymbols() {
456         return restrict_symbols;
457     }
458 
459     ref FilterSymbol getExcludeSymbols() {
460         return exclude_symbols;
461     }
462 
463     ref CompileCommandFilter getCompileCommandFilter() {
464         return compiler_flag_filter;
465     }
466 
467     /// Data produced by the generatore intented to be written to specified file.
468     ref FileData[] getProducedFiles() {
469         return file_data;
470     }
471 
472     void putFile(FileName fname, string data) {
473         file_data ~= FileData(fname, data);
474     }
475 
476     // -- Controller --
477 
478     bool doFile(in string filename, in string info) {
479         import dextool.plugin.regex_matchers : matchAny;
480 
481         bool restrict_pass = true;
482         bool exclude_pass = true;
483 
484         if (restrict.length > 0) {
485             restrict_pass = matchAny(filename, restrict);
486             debug {
487                 logger.tracef(!restrict_pass, "--file-restrict skipping %s", info);
488             }
489         }
490 
491         if (exclude.length > 0) {
492             exclude_pass = !matchAny(filename, exclude);
493             debug {
494                 logger.tracef(!exclude_pass, "--file-exclude skipping %s", info);
495             }
496         }
497 
498         return restrict_pass && exclude_pass;
499     }
500 
501     bool doSymbol(string symbol) {
502         // fast path, assuming no symbol filter is the most common
503         if (!restrict_symbols.hasSymbols && !exclude_symbols.hasSymbols) {
504             return true;
505         }
506 
507         if (restrict_symbols.hasSymbols && exclude_symbols.hasSymbols) {
508             return restrict_symbols.contains(symbol) && !exclude_symbols.contains(symbol);
509         }
510 
511         if (restrict_symbols.hasSymbols) {
512             return restrict_symbols.contains(symbol);
513         }
514 
515         if (exclude_symbols.hasSymbols) {
516             return !exclude_symbols.contains(symbol);
517         }
518 
519         return true;
520     }
521 
522     bool doGoogleMock() {
523         return gmock;
524     }
525 
526     bool doPreIncludes() {
527         import std.file : exists;
528 
529         return pre_incl && !exists(cast(string) pre_incl_file);
530     }
531 
532     bool doIncludeOfPreIncludes() {
533         return pre_incl;
534     }
535 
536     bool doPostIncludes() {
537         import std.file : exists;
538 
539         return post_incl && !exists(cast(string) post_incl_file);
540     }
541 
542     bool doIncludeOfPostIncludes() {
543         return post_incl;
544     }
545 
546     bool doLocationAsComment() {
547         return loc_as_comment;
548     }
549 
550     // -- Parameters --
551 
552     FileName[] getIncludes() {
553         import std.algorithm : map;
554         import std.array : array;
555 
556         return td_includes.includes.map!(a => FileName(a)).array();
557     }
558 
559     DirName getOutputDirectory() {
560         return output_dir;
561     }
562 
563     Parameters.Files getFiles() {
564         return Parameters.Files(main_file_hdr, main_file_impl,
565                 main_file_globals, gmock_file, pre_incl_file, post_incl_file);
566     }
567 
568     MainName getMainName() {
569         return main_name;
570     }
571 
572     MainNs getMainNs() {
573         return main_ns;
574     }
575 
576     MainInterface getMainInterface() {
577         return main_if;
578     }
579 
580     StubPrefix getFilePrefix() {
581         return StubPrefix("");
582     }
583 
584     StubPrefix getArtifactPrefix() {
585         return prefix;
586     }
587 
588     DextoolVersion getToolVersion() {
589         import dextool.utility : dextoolVersion;
590 
591         return dextoolVersion;
592     }
593 
594     CustomHeader getCustomHeader() {
595         return custom_hdr;
596     }
597 
598     Flag!"generateZeroGlobals" generateZeroGlobals() {
599         return generate_zero_globals;
600     }
601 
602     // -- Products --
603 
604     void putFile(FileName fname, CppHModule hdr_data) {
605         file_data ~= FileData(fname, hdr_data.render());
606     }
607 
608     void putFile(FileName fname, CppModule impl_data) {
609         file_data ~= FileData(fname, impl_data.render());
610     }
611 
612     void putLocation(FileName fname, LocationType type) {
613         td_includes.put(fname, type);
614     }
615 }
616 
617 /// TODO refactor, doing too many things.
618 ExitStatusType genCstub(CTestDoubleVariant variant, in string[] in_cflags,
619         CompileCommandDB compile_db, InFiles in_files) {
620     import std.typecons : Yes;
621 
622     import dextool.clang : findFlags;
623     import dextool.compilation_db : ParseData = SearchResult;
624     import cpptooling.analyzer.clang.context : ClangContext;
625     import dextool.io : writeFileData;
626     import dextool.plugin.ctestdouble.backend.cvariant : CVisitor, Generator;
627     import dextool.utility : prependDefaultFlags, PreferLang, analyzeFile;
628 
629     const user_cflags = prependDefaultFlags(in_cflags, PreferLang.c);
630     const total_files = in_files.length;
631     auto visitor = new CVisitor(variant, variant);
632     auto ctx = ClangContext(Yes.useInternalHeaders, Yes.prependParamSyntaxOnly);
633     auto generator = Generator(variant, variant, variant);
634 
635     foreach (idx, in_file; in_files) {
636         logger.infof("File %d/%d ", idx + 1, total_files);
637         ParseData pdata;
638 
639         // TODO duplicate code in c, c++ and plantuml. Fix it.
640         if (compile_db.length > 0) {
641             auto tmp = compile_db.findFlags(FileName(in_file), user_cflags,
642                     variant.getCompileCommandFilter);
643             if (tmp.isNull) {
644                 return ExitStatusType.Errors;
645             }
646             pdata = tmp.get;
647         } else {
648             pdata.flags.prependCflags(user_cflags.dup);
649             pdata.absoluteFile = AbsolutePath(FileName(in_file));
650         }
651 
652         if (analyzeFile(pdata.absoluteFile, pdata.cflags, visitor, ctx) == ExitStatusType.Errors) {
653             return ExitStatusType.Errors;
654         }
655 
656         generator.aggregate(visitor.root, visitor.container);
657         visitor.clearRoot;
658         variant.processIncludes;
659     }
660 
661     variant.finalizeIncludes;
662 
663     // Analyse and generate test double
664     generator.process(visitor.container);
665 
666     debug {
667         logger.trace(visitor);
668     }
669 
670     return writeFileData(variant.getProducedFiles);
671 }