1 /**
2 Date: 2016, Joakim Brännström
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5  */
6 module autobuild;
7 
8 import std.typecons : Flag, Yes, No;
9 
10 import scriptlike;
11 
12 Flag!"SignalInterrupt" signalInterrupt;
13 Flag!"TestsPassed" signalExitStatus;
14 
15 bool skipStaticAnalysis = true;
16 
17 void echoOn() {
18     .scriptlikeEcho = true;
19 }
20 
21 void echoOff() {
22     .scriptlikeEcho = false;
23 }
24 
25 enum Color {
26     red,
27     green,
28     yellow,
29     cancel
30 }
31 
32 enum Status {
33     Fail,
34     Warn,
35     Ok,
36     Run
37 }
38 
39 auto sourcePath() {
40     // those that are supported by the developer of deXtool
41 
42     // dfmt off
43     return only(
44                 "clang",
45                 "dextool_clang_extensions",
46                 "dsrcgen/source",
47                 "llvm_hiwrap/source",
48                 "plugin",
49                 "source",
50                )
51         .map!(a => thisExePath.dirName ~ a)
52         .map!(a => a.toString)
53         .array();
54     // dfmt on
55 }
56 
57 auto gitHEAD() {
58     // Initial commit: diff against an empty tree object
59     string against = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
60 
61     auto res = tryRunCollect("git rev-parse --verify HEAD");
62     if (res.status == 0) {
63         against = res.output;
64     }
65 
66     return against;
67 }
68 
69 auto gitChangdedFiles(string[] file_extensions) {
70     import std.ascii : newline;
71 
72     Args a;
73     a ~= "git";
74     a ~= "diff-index";
75     a ~= "--name-status";
76     a ~= ["--cached", gitHEAD];
77 
78     auto res = tryRunCollect(a.data);
79     if (res.status != 0) {
80         writeln("error: ", res.output);
81     }
82 
83     // dfmt off
84     return res.output
85         .splitter(newline)
86         .map!(a => a.splitter.array())
87         .filter!(a => a.length == 2)
88         .filter!(a => a[0].among("M", "A"))
89         .filter!(a => canFind(file_extensions, std.path.extension(a[1])))
90         .map!(a => a[1]);
91     // dfmt on
92 }
93 
94 auto sourceAsInclude() {
95     // dfmt off
96     return only(
97                 "dsrcgen/source",
98                 "source",
99                 "clang",
100                 "libclang",
101                )
102         .map!(a => thisExePath.dirName ~ a)
103         .map!(a => "-I" ~ a.toString)
104         .array();
105     // dfmt on
106 }
107 
108 auto consoleStaticAnalyse(R)(R lines) {
109     import std.algorithm;
110     import std..string;
111 
112     // dfmt off
113     auto needles = [
114         "taggedalgebraic",
115          "Could not resolve location of module"];
116 
117     return lines
118         // remove those that contains the substrings
119         .filter!((string line) => !any!(a => indexOf(line, a) != -1)(needles))
120         // 15 is arbitrarily chosen
121         .take(15);
122     // dfmt on
123 }
124 
125 void print(T...)(Color c, T args) {
126     static immutable string[] escCodes = ["\033[31;1m", "\033[32;1m", "\033[33;1m", "\033[0;;m"];
127     write(escCodes[c], args, escCodes[Color.cancel]);
128 }
129 
130 void println(T...)(Color c, T args) {
131     static immutable string[] escCodes = ["\033[31;1m", "\033[32;1m", "\033[33;1m", "\033[0;;m"];
132     writeln(escCodes[c], args, escCodes[Color.cancel]);
133 }
134 
135 void printStatus(T...)(Status s, T args) {
136     Color c;
137     string txt;
138 
139     final switch (s) {
140     case Status.Ok:
141         c = Color.green;
142         txt = "[  OK ] ";
143         break;
144     case Status.Run:
145         c = Color.yellow;
146         txt = "[ RUN ] ";
147         break;
148     case Status.Fail:
149         c = Color.red;
150         txt = "[ FAIL] ";
151         break;
152     case Status.Warn:
153         c = Color.red;
154         txt = "[ WARN] ";
155         break;
156     }
157 
158     print(c, txt);
159     writeln(args);
160 }
161 
162 void playSound(Flag!"Positive" positive) nothrow {
163     static import std.stdio;
164     import std.process;
165 
166     static Pid last_pid;
167 
168     try {
169         auto devnull = std.stdio.File("/dev/null", "w");
170 
171         if (last_pid !is null && last_pid.processID != 0) {
172             // cleanup possible zombie process
173             last_pid.wait;
174         }
175 
176         auto a = ["mplayer", "-nostop-xscreensaver"];
177         if (positive)
178             a ~= "/usr/share/sounds/KDE-Sys-App-Positive.ogg";
179         else
180             a ~= "/usr/share/sounds/KDE-Sys-App-Negative.ogg";
181 
182         last_pid = spawnProcess(a, std.stdio.stdin, devnull, devnull);
183     }
184     catch (ProcessException ex) {
185     }
186     catch (Exception ex) {
187     }
188 }
189 
190 bool sanityCheck() {
191     if (!existsAsFile("dub.sdl")) {
192         writeln("Missing dub.sdl");
193         return false;
194     }
195 
196     return true;
197 }
198 
199 void consoleToFile(Path fname, string console) {
200     writeln("console log written to -> ", fname);
201 
202     auto f = File(fname.toString, "w");
203     f.write(console);
204 }
205 
206 Path cmakeDir() {
207     return thisExePath.dirName ~ "build";
208 }
209 
210 void setup() {
211     //echoOn;
212 
213     if (!existsAsDir("build")) {
214         tryRemove("build");
215         mkdir("build");
216     }
217 
218     auto r = tryRunCollect(cmakeDir, "cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TEST=ON ..");
219     writeln(r.output);
220 
221     import core.stdc.signal;
222 
223     signal(SIGINT, &handleSIGINT);
224 }
225 
226 extern (C) void handleSIGINT(int sig) nothrow @nogc @system {
227     .signalInterrupt = Yes.SignalInterrupt;
228 }
229 
230 void cleanup(Flag!"keepCoverage" keep_cov) {
231     import std.algorithm : predSwitch;
232 
233     printStatus(Status.Run, "Cleanup");
234     scope (failure)
235         printStatus(Status.Fail, "Cleanup");
236 
237     // dfmt off
238     chain(
239           dirEntries(".", "trace.*", SpanMode.shallow).map!(a => Path(a)).array(),
240           keep_cov.predSwitch(Yes.keepCoverage, string[].init.map!(a => Path(a)).array(),
241                               No.keepCoverage, dirEntries(".", "*.lst", SpanMode.shallow).map!(a => Path(a)).array())
242          )
243         .each!(a => tryRemove(a));
244     // dfmt on
245 
246     printStatus(Status.Ok, "Cleanup");
247 }
248 
249 /** Call appropriate function for for the state.
250  *
251  * Generate calls to functions of fsm based on st.
252  *
253  * Params:
254  *  fsm = object with methods with prefix st_
255  *  st = current state
256  */
257 auto GenerateFsmAction(T, TEnum)(ref T fsm, TEnum st) {
258     import std.traits;
259 
260     final switch (st) {
261         foreach (e; EnumMembers!TEnum) {
262             mixin(format(q{
263                          case %s.%s.%s:
264                            fsm.state%s();
265                            break;
266 
267                          }, typeof(fsm).stringof, TEnum.stringof, e, e));
268         }
269     }
270 }
271 
272 /// Moore FSM
273 /// Exceptions are clearly documented with // FSM exception: REASON
274 struct Fsm {
275     enum State {
276         Init,
277         Reset,
278         Wait,
279         Start,
280         Ut_run,
281         Ut_skip,
282         Debug_build,
283         Integration_test,
284         Test_passed,
285         Test_failed,
286         Doc_check_counter,
287         Doc_build,
288         Sloc_check_counter,
289         Slocs,
290         AudioStatus,
291         ExitOrRestart,
292         Exit
293     }
294 
295     State st;
296     Path[] inotify_paths;
297 
298     Flag!"utDebug" flagUtDebug;
299 
300     // Signals used to determine next state
301     Flag!"UtTestPassed" flagUtTestPassed;
302     Flag!"CompileError" flagCompileError;
303     Flag!"TotalTestPassed" flagTotalTestPassed;
304     int docCount;
305     int analyzeCount;
306 
307     alias ErrorMsg = Tuple!(Path, "fname", string, "msg", string, "output");
308     ErrorMsg[] testErrorLog;
309 
310     void run(Path[] inotify_paths, Flag!"Travis" travis,
311             Flag!"utDebug" ut_debug, Flag!"utSkip" ut_skip) {
312         this.inotify_paths = inotify_paths;
313         this.flagUtDebug = ut_debug;
314 
315         while (!signalInterrupt) {
316             debug {
317                 writeln("State ", st.to!string);
318             }
319 
320             GenerateFsmAction(this, st);
321 
322             updateTotalTestStatus();
323 
324             st = Fsm.next(st, docCount, analyzeCount, flagUtTestPassed,
325                     flagCompileError, flagTotalTestPassed, travis, ut_skip);
326         }
327     }
328 
329     void updateTotalTestStatus() {
330         if (testErrorLog.length != 0) {
331             flagTotalTestPassed = No.TotalTestPassed;
332         } else if (flagUtTestPassed == No.UtTestPassed) {
333             flagTotalTestPassed = No.TotalTestPassed;
334         } else if (flagCompileError == Yes.CompileError) {
335             flagTotalTestPassed = No.TotalTestPassed;
336         } else {
337             flagTotalTestPassed = Yes.TotalTestPassed;
338         }
339     }
340 
341     static State next(State st, int docCount, int analyzeCount,
342             Flag!"UtTestPassed" flagUtTestPassed, Flag!"CompileError" flagCompileError,
343             Flag!"TotalTestPassed" flagTotalTestPassed, Flag!"Travis" travis,
344             Flag!"utSkip" ut_skip) {
345         auto next_ = st;
346 
347         final switch (st) {
348         case State.Init:
349             next_ = State.Start;
350             break;
351         case State.AudioStatus:
352             next_ = State.Reset;
353             break;
354         case State.Reset:
355             next_ = State.Wait;
356             break;
357         case State.Wait:
358             next_ = State.Start;
359             break;
360         case State.Start:
361             next_ = State.Ut_run;
362             if (ut_skip) {
363                 next_ = State.Ut_skip;
364             }
365             break;
366         case State.Ut_run:
367             next_ = State.ExitOrRestart;
368             if (flagUtTestPassed)
369                 next_ = State.Debug_build;
370             break;
371         case State.Ut_skip:
372             next_ = State.Debug_build;
373             break;
374         case State.Debug_build:
375             next_ = State.Integration_test;
376             if (flagCompileError)
377                 next_ = State.ExitOrRestart;
378             break;
379         case State.Integration_test:
380             next_ = State.ExitOrRestart;
381             if (flagTotalTestPassed)
382                 next_ = State.Test_passed;
383             else
384                 next_ = State.Test_failed;
385             break;
386         case State.Test_passed:
387             next_ = State.Doc_check_counter;
388             break;
389         case State.Test_failed:
390             next_ = State.ExitOrRestart;
391             break;
392         case State.Doc_check_counter:
393             next_ = State.ExitOrRestart;
394             if (docCount >= 10 && !travis)
395                 next_ = State.Doc_build;
396             break;
397         case State.Doc_build:
398             next_ = State.Sloc_check_counter;
399             break;
400         case State.Sloc_check_counter:
401             next_ = State.ExitOrRestart;
402             if (analyzeCount >= 10) {
403                 next_ = State.Slocs;
404             }
405             break;
406         case State.Slocs:
407             next_ = State.ExitOrRestart;
408             break;
409         case State.ExitOrRestart:
410             next_ = State.AudioStatus;
411             if (travis) {
412                 next_ = State.Exit;
413             }
414             break;
415         case State.Exit:
416             break;
417         }
418 
419         return next_;
420     }
421 
422     static void printExitStatus(T...)(int status, T args) {
423         if (status == 0)
424             printStatus(Status.Ok, args);
425         else
426             printStatus(Status.Fail, args);
427     }
428 
429     void stateInit() {
430         // force rebuild of doc and show code stat
431         docCount = 10;
432         analyzeCount = 10;
433 
434         writeln("Watching the following paths for changes:");
435         inotify_paths.each!writeln;
436     }
437 
438     void stateAudioStatus() {
439         if (!flagCompileError && flagUtTestPassed && testErrorLog.length == 0)
440             playSound(Yes.Positive);
441         else
442             playSound(No.Positive);
443     }
444 
445     void stateReset() {
446         flagCompileError = No.CompileError;
447         flagUtTestPassed = No.UtTestPassed;
448         testErrorLog.length = 0;
449     }
450 
451     void stateStart() {
452     }
453 
454     void stateWait() {
455         println(Color.yellow, "================================");
456 
457         Args a;
458         a ~= "inotifywait";
459         a ~= "-q";
460         a ~= "-r";
461         a ~= ["-e", "modify"];
462         a ~= ["-e", "attrib"];
463         a ~= ["-e", "create"];
464         a ~= ["-e", "move_self"];
465         a ~= ["--format", "%w"];
466         a ~= inotify_paths;
467 
468         auto r = tryRunCollect(thisExePath.dirName, a.data);
469 
470         import core.thread;
471 
472         if (signalInterrupt) {
473             // do nothing, a SIGINT has been received while sleeping
474         } else if (r.status == 0) {
475             writeln("Change detected in ", r.output);
476             // wait for editor to finish saving the file
477             Thread.sleep(dur!("msecs")(500));
478         } else {
479             enum SLEEP = 10;
480             writeln(a.data);
481             printStatus(Status.Warn, "Error: ", r.output);
482             writeln("sleeping ", SLEEP, "s");
483             Thread.sleep(dur!("seconds")(SLEEP));
484         }
485     }
486 
487     void stateUt_run() {
488         immutable test_header = "Compile and run unittest";
489         printStatus(Status.Run, test_header);
490 
491         auto status = tryRun(cmakeDir, "make check -j $(nproc)");
492         flagUtTestPassed = cast(Flag!"UtTestPassed")(status == 0);
493 
494         printExitStatus(status, test_header);
495     }
496 
497     void stateUt_skip() {
498         flagUtTestPassed = Yes.UtTestPassed;
499     }
500 
501     void stateDebug_build() {
502         printStatus(Status.Run, "Debug build");
503 
504         auto r = tryRun(cmakeDir, "make all -j $(nproc)");
505         flagCompileError = cast(Flag!"CompileError")(r != 0);
506         printExitStatus(r, "Debug build with debug symbols");
507     }
508 
509     void stateIntegration_test() {
510         immutable test_header = "Compile and run integration tests";
511         printStatus(Status.Run, test_header);
512 
513         auto status = tryRun(cmakeDir, `make check_integration -j $(nproc)`);
514 
515         if (status != 0) {
516             testErrorLog ~= ErrorMsg(cmakeDir, "integration_test", "failed");
517         }
518 
519         printExitStatus(status, test_header);
520     }
521 
522     void stateTest_passed() {
523         docCount++;
524         analyzeCount++;
525         printStatus(Status.Ok, "Test of code generation");
526     }
527 
528     void stateTest_failed() {
529         // separate the log dump to the console from the list of files the logs can be found in.
530         // Most common scenario is one failure.
531         testErrorLog.each!((a) { writeln(a.output); });
532         testErrorLog.each!((a) {
533             printStatus(Status.Fail, a.msg, ", log at ", a.fname);
534         });
535 
536         printStatus(Status.Fail, "Test of code generation");
537     }
538 
539     void stateDoc_check_counter() {
540     }
541 
542     void stateDoc_build() {
543     }
544 
545     void stateSloc_check_counter() {
546     }
547 
548     void stateSlocs() {
549         printStatus(Status.Run, "Code statistics");
550         scope (exit)
551             printStatus(Status.Ok, "Code statistics");
552 
553         Args a;
554         a ~= "dscanner";
555         a ~= "--sloc";
556         a ~= sourcePath.array();
557 
558         auto r = tryRunCollect(thisExePath.dirName, a.data);
559         if (r.status == 0) {
560             writeln(r.output);
561         }
562 
563         analyzeCount = 0;
564     }
565 
566     void stateStaticAnalyse() {
567         static import std.file;
568 
569         static import std.stdio;
570 
571         static import core.stdc.stdlib;
572 
573         printStatus(Status.Run, "Static analyze");
574 
575         // dscanner hangs during analysis
576         if (skipStaticAnalysis)
577             return;
578 
579         string phobos_path = core.stdc.stdlib.getenv("DLANG_PHOBOS_PATH".toStringz)
580             .fromStringz.idup;
581         string druntime_path = core.stdc.stdlib.getenv("DLANG_DRUNTIME_PATH".toStringz)
582             .fromStringz.idup;
583 
584         Args a;
585         a ~= "dscanner";
586         a ~= ["--config", (thisExePath.dirName ~ ".dscanner.ini").toString];
587         a ~= "--styleCheck";
588         a ~= "--skipTests";
589 
590         if (phobos_path.length > 0 && druntime_path.length > 0) {
591             a ~= ["-I", phobos_path];
592             a ~= ["-I", druntime_path];
593         } else {
594             println(Color.red, "Extra errors during static analyze");
595             println(Color.red, "Missing env variable DLANG_PHOBOS_PATH and/or DLANG_DRUNTIME_PATH");
596         }
597 
598         a ~= sourceAsInclude;
599         a ~= gitChangdedFiles([".d"]);
600 
601         auto r = tryRunCollect(thisExePath.dirName, a.data);
602 
603         string reportFile = (thisExePath.dirName ~ "dscanner_report.txt").toString;
604         if (r.status != 0) {
605             auto lines = r.output.splitter("\n");
606             const auto dscanner_count = lines.save.count;
607 
608             // console dump
609             consoleStaticAnalyse(lines.save).each!writeln;
610 
611             // dump to file
612             auto fout = File(reportFile, "w");
613             fout.write(a.data ~ "\n" ~ r.output);
614 
615             printStatus(Status.Fail, "Static analyze failed. Found ",
616                     dscanner_count, " error(s). See report ", reportFile);
617         } else {
618             tryRemove(reportFile);
619             printStatus(Status.Ok, "Static analysis");
620         }
621     }
622 
623     void stateExitOrRestart() {
624     }
625 
626     void stateExit() {
627         if (flagTotalTestPassed) {
628             .signalExitStatus = Yes.TestsPassed;
629         } else {
630             .signalExitStatus = No.TestsPassed;
631         }
632         .signalInterrupt = Yes.SignalInterrupt;
633     }
634 }
635 
636 int main(string[] args) {
637     Flag!"keepCoverage" keep_cov;
638 
639     chdir(thisExePath.dirName);
640     scope (exit)
641         cleanup(keep_cov);
642 
643     if (!sanityCheck) {
644         writeln("error: Sanity check failed");
645         return 1;
646     }
647 
648     import std.getopt;
649 
650     bool run_and_exit;
651     bool ut_debug;
652     bool ut_skip;
653 
654     // dfmt off
655     auto help_info = getopt(args,
656         "run_and_exit", "run the tests in one pass and exit", &run_and_exit,
657         "ut_debug", "run tests in single threaded debug mode", &ut_debug,
658         "ut_skip", "skip unittests to go straight to the integration tests", &ut_skip);
659     // dfmt on
660 
661     if (help_info.helpWanted) {
662         defaultGetoptPrinter("Usage: autobuild.sh [options]", help_info.options);
663         return 0;
664     }
665 
666     setup();
667 
668     // dfmt off
669     auto inotify_paths = only(
670                               "source",
671                               "plugin",
672                               "clang",
673                               "dextool_clang_extensions",
674                               "libclang",
675                               "dub.sdl",
676                               "dsrcgen/source",
677                               "llvm_hiwrap",
678                               "test/testdata",
679                               "vendor",
680                               "test/source"
681         )
682         .map!(a => thisExePath.dirName ~ a)
683         .array;
684     // dfmt on
685 
686     import std.stdio;
687 
688     (Fsm()).run(inotify_paths, cast(Flag!"Travis") run_and_exit,
689             cast(Flag!"utDebug") ut_debug, cast(Flag!"utSkip") ut_skip);
690 
691     return signalExitStatus ? 0 : -1;
692 }