1 /** 2 Date: 2016, Joakim Brännström 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module autobuild; 7 8 import std.typecons : Flag, Yes, No; 9 10 import scriptlike; 11 12 Flag!"SignalInterrupt" signalInterrupt; 13 Flag!"TestsPassed" signalExitStatus; 14 15 bool skipStaticAnalysis = true; 16 17 void echoOn() { 18 .scriptlikeEcho = true; 19 } 20 21 void echoOff() { 22 .scriptlikeEcho = false; 23 } 24 25 enum Color { 26 red, 27 green, 28 yellow, 29 cancel 30 } 31 32 enum Status { 33 Fail, 34 Warn, 35 Ok, 36 Run 37 } 38 39 auto sourcePath() { 40 // those that are supported by the developer of deXtool 41 42 // dfmt off 43 return only( 44 "clang", 45 "dextool_clang_extensions", 46 "dsrcgen/source", 47 "llvm_hiwrap/source", 48 "plugin", 49 "source", 50 ) 51 .map!(a => thisExePath.dirName ~ a) 52 .map!(a => a.toString) 53 .array(); 54 // dfmt on 55 } 56 57 auto gitHEAD() { 58 // Initial commit: diff against an empty tree object 59 string against = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; 60 61 auto res = tryRunCollect("git rev-parse --verify HEAD"); 62 if (res.status == 0) { 63 against = res.output; 64 } 65 66 return against; 67 } 68 69 auto gitChangdedFiles(string[] file_extensions) { 70 import std.ascii : newline; 71 72 Args a; 73 a ~= "git"; 74 a ~= "diff-index"; 75 a ~= "--name-status"; 76 a ~= ["--cached", gitHEAD]; 77 78 auto res = tryRunCollect(a.data); 79 if (res.status != 0) { 80 writeln("error: ", res.output); 81 } 82 83 // dfmt off 84 return res.output 85 .splitter(newline) 86 .map!(a => a.splitter.array()) 87 .filter!(a => a.length == 2) 88 .filter!(a => a[0].among("M", "A")) 89 .filter!(a => canFind(file_extensions, std.path.extension(a[1]))) 90 .map!(a => a[1]); 91 // dfmt on 92 } 93 94 auto sourceAsInclude() { 95 // dfmt off 96 return only( 97 "dsrcgen/source", 98 "source", 99 "clang", 100 "libclang", 101 ) 102 .map!(a => thisExePath.dirName ~ a) 103 .map!(a => "-I" ~ a.toString) 104 .array(); 105 // dfmt on 106 } 107 108 auto consoleStaticAnalyse(R)(R lines) { 109 import std.algorithm; 110 import std..string; 111 112 // dfmt off 113 auto needles = [ 114 "taggedalgebraic", 115 "Could not resolve location of module"]; 116 117 return lines 118 // remove those that contains the substrings 119 .filter!((string line) => !any!(a => indexOf(line, a) != -1)(needles)) 120 // 15 is arbitrarily chosen 121 .take(15); 122 // dfmt on 123 } 124 125 void print(T...)(Color c, T args) { 126 static immutable string[] escCodes = ["\033[31;1m", "\033[32;1m", "\033[33;1m", "\033[0;;m"]; 127 write(escCodes[c], args, escCodes[Color.cancel]); 128 } 129 130 void println(T...)(Color c, T args) { 131 static immutable string[] escCodes = ["\033[31;1m", "\033[32;1m", "\033[33;1m", "\033[0;;m"]; 132 writeln(escCodes[c], args, escCodes[Color.cancel]); 133 } 134 135 void printStatus(T...)(Status s, T args) { 136 Color c; 137 string txt; 138 139 final switch (s) { 140 case Status.Ok: 141 c = Color.green; 142 txt = "[ OK ] "; 143 break; 144 case Status.Run: 145 c = Color.yellow; 146 txt = "[ RUN ] "; 147 break; 148 case Status.Fail: 149 c = Color.red; 150 txt = "[ FAIL] "; 151 break; 152 case Status.Warn: 153 c = Color.red; 154 txt = "[ WARN] "; 155 break; 156 } 157 158 print(c, txt); 159 writeln(args); 160 } 161 162 void playSound(Flag!"Positive" positive) nothrow { 163 static import std.stdio; 164 import std.process; 165 166 static Pid last_pid; 167 168 try { 169 auto devnull = std.stdio.File("/dev/null", "w"); 170 171 if (last_pid !is null && last_pid.processID != 0) { 172 // cleanup possible zombie process 173 last_pid.wait; 174 } 175 176 auto a = ["mplayer", "-nostop-xscreensaver"]; 177 if (positive) 178 a ~= "/usr/share/sounds/KDE-Sys-App-Positive.ogg"; 179 else 180 a ~= "/usr/share/sounds/KDE-Sys-App-Negative.ogg"; 181 182 last_pid = spawnProcess(a, std.stdio.stdin, devnull, devnull); 183 } 184 catch (ProcessException ex) { 185 } 186 catch (Exception ex) { 187 } 188 } 189 190 bool sanityCheck() { 191 if (!existsAsFile("dub.sdl")) { 192 writeln("Missing dub.sdl"); 193 return false; 194 } 195 196 return true; 197 } 198 199 void consoleToFile(Path fname, string console) { 200 writeln("console log written to -> ", fname); 201 202 auto f = File(fname.toString, "w"); 203 f.write(console); 204 } 205 206 Path cmakeDir() { 207 return thisExePath.dirName ~ "build"; 208 } 209 210 void setup() { 211 //echoOn; 212 213 if (!existsAsDir("build")) { 214 tryRemove("build"); 215 mkdir("build"); 216 } 217 218 auto r = tryRunCollect(cmakeDir, "cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TEST=ON .."); 219 writeln(r.output); 220 221 import core.stdc.signal; 222 223 signal(SIGINT, &handleSIGINT); 224 } 225 226 extern (C) void handleSIGINT(int sig) nothrow @nogc @system { 227 .signalInterrupt = Yes.SignalInterrupt; 228 } 229 230 void cleanup(Flag!"keepCoverage" keep_cov) { 231 import std.algorithm : predSwitch; 232 233 printStatus(Status.Run, "Cleanup"); 234 scope (failure) 235 printStatus(Status.Fail, "Cleanup"); 236 237 // dfmt off 238 chain( 239 dirEntries(".", "trace.*", SpanMode.shallow).map!(a => Path(a)).array(), 240 keep_cov.predSwitch(Yes.keepCoverage, string[].init.map!(a => Path(a)).array(), 241 No.keepCoverage, dirEntries(".", "*.lst", SpanMode.shallow).map!(a => Path(a)).array()) 242 ) 243 .each!(a => tryRemove(a)); 244 // dfmt on 245 246 printStatus(Status.Ok, "Cleanup"); 247 } 248 249 /** Call appropriate function for for the state. 250 * 251 * Generate calls to functions of fsm based on st. 252 * 253 * Params: 254 * fsm = object with methods with prefix st_ 255 * st = current state 256 */ 257 auto GenerateFsmAction(T, TEnum)(ref T fsm, TEnum st) { 258 import std.traits; 259 260 final switch (st) { 261 foreach (e; EnumMembers!TEnum) { 262 mixin(format(q{ 263 case %s.%s.%s: 264 fsm.state%s(); 265 break; 266 267 }, typeof(fsm).stringof, TEnum.stringof, e, e)); 268 } 269 } 270 } 271 272 /// Moore FSM 273 /// Exceptions are clearly documented with // FSM exception: REASON 274 struct Fsm { 275 enum State { 276 Init, 277 Reset, 278 Wait, 279 Start, 280 Ut_run, 281 Ut_skip, 282 Debug_build, 283 Integration_test, 284 Test_passed, 285 Test_failed, 286 Doc_check_counter, 287 Doc_build, 288 Sloc_check_counter, 289 Slocs, 290 AudioStatus, 291 ExitOrRestart, 292 Exit 293 } 294 295 State st; 296 Path[] inotify_paths; 297 298 Flag!"utDebug" flagUtDebug; 299 300 // Signals used to determine next state 301 Flag!"UtTestPassed" flagUtTestPassed; 302 Flag!"CompileError" flagCompileError; 303 Flag!"TotalTestPassed" flagTotalTestPassed; 304 int docCount; 305 int analyzeCount; 306 307 alias ErrorMsg = Tuple!(Path, "fname", string, "msg", string, "output"); 308 ErrorMsg[] testErrorLog; 309 310 void run(Path[] inotify_paths, Flag!"Travis" travis, 311 Flag!"utDebug" ut_debug, Flag!"utSkip" ut_skip) { 312 this.inotify_paths = inotify_paths; 313 this.flagUtDebug = ut_debug; 314 315 while (!signalInterrupt) { 316 debug { 317 writeln("State ", st.to!string); 318 } 319 320 GenerateFsmAction(this, st); 321 322 updateTotalTestStatus(); 323 324 st = Fsm.next(st, docCount, analyzeCount, flagUtTestPassed, 325 flagCompileError, flagTotalTestPassed, travis, ut_skip); 326 } 327 } 328 329 void updateTotalTestStatus() { 330 if (testErrorLog.length != 0) { 331 flagTotalTestPassed = No.TotalTestPassed; 332 } else if (flagUtTestPassed == No.UtTestPassed) { 333 flagTotalTestPassed = No.TotalTestPassed; 334 } else if (flagCompileError == Yes.CompileError) { 335 flagTotalTestPassed = No.TotalTestPassed; 336 } else { 337 flagTotalTestPassed = Yes.TotalTestPassed; 338 } 339 } 340 341 static State next(State st, int docCount, int analyzeCount, 342 Flag!"UtTestPassed" flagUtTestPassed, Flag!"CompileError" flagCompileError, 343 Flag!"TotalTestPassed" flagTotalTestPassed, Flag!"Travis" travis, 344 Flag!"utSkip" ut_skip) { 345 auto next_ = st; 346 347 final switch (st) { 348 case State.Init: 349 next_ = State.Start; 350 break; 351 case State.AudioStatus: 352 next_ = State.Reset; 353 break; 354 case State.Reset: 355 next_ = State.Wait; 356 break; 357 case State.Wait: 358 next_ = State.Start; 359 break; 360 case State.Start: 361 next_ = State.Ut_run; 362 if (ut_skip) { 363 next_ = State.Ut_skip; 364 } 365 break; 366 case State.Ut_run: 367 next_ = State.ExitOrRestart; 368 if (flagUtTestPassed) 369 next_ = State.Debug_build; 370 break; 371 case State.Ut_skip: 372 next_ = State.Debug_build; 373 break; 374 case State.Debug_build: 375 next_ = State.Integration_test; 376 if (flagCompileError) 377 next_ = State.ExitOrRestart; 378 break; 379 case State.Integration_test: 380 next_ = State.ExitOrRestart; 381 if (flagTotalTestPassed) 382 next_ = State.Test_passed; 383 else 384 next_ = State.Test_failed; 385 break; 386 case State.Test_passed: 387 next_ = State.Doc_check_counter; 388 break; 389 case State.Test_failed: 390 next_ = State.ExitOrRestart; 391 break; 392 case State.Doc_check_counter: 393 next_ = State.ExitOrRestart; 394 if (docCount >= 10 && !travis) 395 next_ = State.Doc_build; 396 break; 397 case State.Doc_build: 398 next_ = State.Sloc_check_counter; 399 break; 400 case State.Sloc_check_counter: 401 next_ = State.ExitOrRestart; 402 if (analyzeCount >= 10) { 403 next_ = State.Slocs; 404 } 405 break; 406 case State.Slocs: 407 next_ = State.ExitOrRestart; 408 break; 409 case State.ExitOrRestart: 410 next_ = State.AudioStatus; 411 if (travis) { 412 next_ = State.Exit; 413 } 414 break; 415 case State.Exit: 416 break; 417 } 418 419 return next_; 420 } 421 422 static void printExitStatus(T...)(int status, T args) { 423 if (status == 0) 424 printStatus(Status.Ok, args); 425 else 426 printStatus(Status.Fail, args); 427 } 428 429 void stateInit() { 430 // force rebuild of doc and show code stat 431 docCount = 10; 432 analyzeCount = 10; 433 434 writeln("Watching the following paths for changes:"); 435 inotify_paths.each!writeln; 436 } 437 438 void stateAudioStatus() { 439 if (!flagCompileError && flagUtTestPassed && testErrorLog.length == 0) 440 playSound(Yes.Positive); 441 else 442 playSound(No.Positive); 443 } 444 445 void stateReset() { 446 flagCompileError = No.CompileError; 447 flagUtTestPassed = No.UtTestPassed; 448 testErrorLog.length = 0; 449 } 450 451 void stateStart() { 452 } 453 454 void stateWait() { 455 println(Color.yellow, "================================"); 456 457 Args a; 458 a ~= "inotifywait"; 459 a ~= "-q"; 460 a ~= "-r"; 461 a ~= ["-e", "modify"]; 462 a ~= ["-e", "attrib"]; 463 a ~= ["-e", "create"]; 464 a ~= ["-e", "move_self"]; 465 a ~= ["--format", "%w"]; 466 a ~= inotify_paths; 467 468 auto r = tryRunCollect(thisExePath.dirName, a.data); 469 470 import core.thread; 471 472 if (signalInterrupt) { 473 // do nothing, a SIGINT has been received while sleeping 474 } else if (r.status == 0) { 475 writeln("Change detected in ", r.output); 476 // wait for editor to finish saving the file 477 Thread.sleep(dur!("msecs")(500)); 478 } else { 479 enum SLEEP = 10; 480 writeln(a.data); 481 printStatus(Status.Warn, "Error: ", r.output); 482 writeln("sleeping ", SLEEP, "s"); 483 Thread.sleep(dur!("seconds")(SLEEP)); 484 } 485 } 486 487 void stateUt_run() { 488 immutable test_header = "Compile and run unittest"; 489 printStatus(Status.Run, test_header); 490 491 auto status = tryRun(cmakeDir, "make check -j $(nproc)"); 492 flagUtTestPassed = cast(Flag!"UtTestPassed")(status == 0); 493 494 printExitStatus(status, test_header); 495 } 496 497 void stateUt_skip() { 498 flagUtTestPassed = Yes.UtTestPassed; 499 } 500 501 void stateDebug_build() { 502 printStatus(Status.Run, "Debug build"); 503 504 auto r = tryRun(cmakeDir, "make all -j $(nproc)"); 505 flagCompileError = cast(Flag!"CompileError")(r != 0); 506 printExitStatus(r, "Debug build with debug symbols"); 507 } 508 509 void stateIntegration_test() { 510 immutable test_header = "Compile and run integration tests"; 511 printStatus(Status.Run, test_header); 512 513 auto status = tryRun(cmakeDir, `make check_integration -j $(nproc)`); 514 515 if (status != 0) { 516 testErrorLog ~= ErrorMsg(cmakeDir, "integration_test", "failed"); 517 } 518 519 printExitStatus(status, test_header); 520 } 521 522 void stateTest_passed() { 523 docCount++; 524 analyzeCount++; 525 printStatus(Status.Ok, "Test of code generation"); 526 } 527 528 void stateTest_failed() { 529 // separate the log dump to the console from the list of files the logs can be found in. 530 // Most common scenario is one failure. 531 testErrorLog.each!((a) { writeln(a.output); }); 532 testErrorLog.each!((a) { 533 printStatus(Status.Fail, a.msg, ", log at ", a.fname); 534 }); 535 536 printStatus(Status.Fail, "Test of code generation"); 537 } 538 539 void stateDoc_check_counter() { 540 } 541 542 void stateDoc_build() { 543 } 544 545 void stateSloc_check_counter() { 546 } 547 548 void stateSlocs() { 549 printStatus(Status.Run, "Code statistics"); 550 scope (exit) 551 printStatus(Status.Ok, "Code statistics"); 552 553 Args a; 554 a ~= "dscanner"; 555 a ~= "--sloc"; 556 a ~= sourcePath.array(); 557 558 auto r = tryRunCollect(thisExePath.dirName, a.data); 559 if (r.status == 0) { 560 writeln(r.output); 561 } 562 563 analyzeCount = 0; 564 } 565 566 void stateStaticAnalyse() { 567 static import std.file; 568 569 static import std.stdio; 570 571 static import core.stdc.stdlib; 572 573 printStatus(Status.Run, "Static analyze"); 574 575 // dscanner hangs during analysis 576 if (skipStaticAnalysis) 577 return; 578 579 string phobos_path = core.stdc.stdlib.getenv("DLANG_PHOBOS_PATH".toStringz) 580 .fromStringz.idup; 581 string druntime_path = core.stdc.stdlib.getenv("DLANG_DRUNTIME_PATH".toStringz) 582 .fromStringz.idup; 583 584 Args a; 585 a ~= "dscanner"; 586 a ~= ["--config", (thisExePath.dirName ~ ".dscanner.ini").toString]; 587 a ~= "--styleCheck"; 588 a ~= "--skipTests"; 589 590 if (phobos_path.length > 0 && druntime_path.length > 0) { 591 a ~= ["-I", phobos_path]; 592 a ~= ["-I", druntime_path]; 593 } else { 594 println(Color.red, "Extra errors during static analyze"); 595 println(Color.red, "Missing env variable DLANG_PHOBOS_PATH and/or DLANG_DRUNTIME_PATH"); 596 } 597 598 a ~= sourceAsInclude; 599 a ~= gitChangdedFiles([".d"]); 600 601 auto r = tryRunCollect(thisExePath.dirName, a.data); 602 603 string reportFile = (thisExePath.dirName ~ "dscanner_report.txt").toString; 604 if (r.status != 0) { 605 auto lines = r.output.splitter("\n"); 606 const auto dscanner_count = lines.save.count; 607 608 // console dump 609 consoleStaticAnalyse(lines.save).each!writeln; 610 611 // dump to file 612 auto fout = File(reportFile, "w"); 613 fout.write(a.data ~ "\n" ~ r.output); 614 615 printStatus(Status.Fail, "Static analyze failed. Found ", 616 dscanner_count, " error(s). See report ", reportFile); 617 } else { 618 tryRemove(reportFile); 619 printStatus(Status.Ok, "Static analysis"); 620 } 621 } 622 623 void stateExitOrRestart() { 624 } 625 626 void stateExit() { 627 if (flagTotalTestPassed) { 628 .signalExitStatus = Yes.TestsPassed; 629 } else { 630 .signalExitStatus = No.TestsPassed; 631 } 632 .signalInterrupt = Yes.SignalInterrupt; 633 } 634 } 635 636 int main(string[] args) { 637 Flag!"keepCoverage" keep_cov; 638 639 chdir(thisExePath.dirName); 640 scope (exit) 641 cleanup(keep_cov); 642 643 if (!sanityCheck) { 644 writeln("error: Sanity check failed"); 645 return 1; 646 } 647 648 import std.getopt; 649 650 bool run_and_exit; 651 bool ut_debug; 652 bool ut_skip; 653 654 // dfmt off 655 auto help_info = getopt(args, 656 "run_and_exit", "run the tests in one pass and exit", &run_and_exit, 657 "ut_debug", "run tests in single threaded debug mode", &ut_debug, 658 "ut_skip", "skip unittests to go straight to the integration tests", &ut_skip); 659 // dfmt on 660 661 if (help_info.helpWanted) { 662 defaultGetoptPrinter("Usage: autobuild.sh [options]", help_info.options); 663 return 0; 664 } 665 666 setup(); 667 668 // dfmt off 669 auto inotify_paths = only( 670 "source", 671 "plugin", 672 "clang", 673 "dextool_clang_extensions", 674 "libclang", 675 "dub.sdl", 676 "dsrcgen/source", 677 "llvm_hiwrap", 678 "test/testdata", 679 "vendor", 680 "test/source" 681 ) 682 .map!(a => thisExePath.dirName ~ a) 683 .array; 684 // dfmt on 685 686 import std.stdio; 687 688 (Fsm()).run(inotify_paths, cast(Flag!"Travis") run_and_exit, 689 cast(Flag!"utDebug") ut_debug, cast(Flag!"utSkip") ut_skip); 690 691 return signalExitStatus ? 0 : -1; 692 }