1 /**
2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool_test.golden;
11 
12 import scriptlike;
13 
14 struct BuildCompare {
15     import std.typecons : Yes, No;
16 
17     private {
18         string outdir_;
19 
20         Flag!"sortLines" sort_lines = No.sortLines;
21         Flag!"skipComments" skip_comments = Yes.skipComments;
22 
23         /// if the output from running the command should be yapped via scriptlike
24         bool yap_output = true;
25 
26         /// Throw an exception if a compare failes
27         bool throw_on_failed_compare_ = true;
28 
29         GoldResult[] gold_results;
30     }
31 
32     private static struct GoldResult {
33         Path gold;
34         Path result;
35     }
36 
37     this(string outdir) {
38         this.outdir_ = outdir;
39     }
40 
41     Path outdir() {
42         return Path(outdir_);
43     }
44 
45     auto addCompare(Path gold, string result_file) {
46         this.gold_results ~= GoldResult(gold, buildPath(outdir_, result_file).Path);
47         return this;
48     }
49 
50     auto sortLines(bool v) {
51         sort_lines = cast(Flag!"sortLines") v;
52         return this;
53     }
54 
55     auto skipComments(bool v) {
56         skip_comments = cast(Flag!"skipComments") v;
57         return this;
58     }
59 
60     auto throwOnFailure(bool v) {
61         this.throw_on_failed_compare_ = v;
62         return this;
63     }
64 
65     auto run() {
66         CompareResult res;
67 
68         foreach (const ref gr; gold_results) {
69             res = compare(gr.gold, gr.result, sort_lines, skip_comments);
70             if (!res.status) {
71                 break;
72             }
73         }
74 
75         if (!res.status && yap_output) {
76             File(nextFreeLogfile(outdir_), "w").writef("%s", res);
77         }
78 
79         if (!res.status && throw_on_failed_compare_) {
80             throw new ErrorLevelException(1, res.errorMsg);
81         }
82 
83         return res;
84     }
85 }
86 
87 /** Sorted compare of gold and result.
88  *
89  * max_diff is arbitrarily chosen to 5.
90  * The purpose is to limit the amount of text that is dumped.
91  * The reasoning is that it is better to give more than one line as feedback.
92  */
93 private CompareResult compare(const Path gold, const Path result,
94         Flag!"sortLines" sortLines, Flag!"skipComments" skipComments) {
95     import std.format : format;
96     import std.stdio : File;
97     import dextool_test.utils : escapePath, removeJunk;
98 
99     CompareResult res;
100 
101     res.msg ~= "Comparing gold:" ~ gold.raw;
102     res.msg ~= "        result:" ~ result.raw;
103 
104     File goldf;
105     File resultf;
106 
107     try {
108         goldf = File(gold.escapePath);
109         resultf = File(result.escapePath);
110     }
111     catch (ErrnoException ex) {
112         res.errorMsg = ex.msg;
113         res.status = false;
114         return res;
115     }
116 
117     auto maybeSort(T)(T lines) {
118         import std.array : array;
119         import std.algorithm : sort;
120 
121         if (sortLines) {
122             return sort!((a, b) => a[1] < b[1])(lines.array()).array();
123         }
124 
125         return lines.array();
126     }
127 
128     bool diff_detected = false;
129     immutable max_diff = 5;
130     int accumulated_diff;
131     // dfmt off
132     foreach (g, r;
133              lockstep(maybeSort(goldf
134                                 .byLineCopy()
135                                 .enumerate
136                                 .removeJunk(skipComments)),
137                       maybeSort(resultf
138                                 .byLineCopy()
139                                 .enumerate
140                                 .removeJunk(skipComments))
141                       )) {
142         if (g[1] != r[1] && accumulated_diff < max_diff) {
143             // +1 of index because editors start counting lines from 1
144             res.lineDiff ~= format("Line %s gold: %s", g[0] + 1, g[1]);
145             res.lineDiff ~= format("Line %s  out: %s", r[0] + 1, r[1]);
146             diff_detected = true;
147             ++accumulated_diff;
148         }
149     }
150     // dfmt on
151 
152     res.status = !diff_detected;
153 
154     if (diff_detected) {
155         res.errorMsg = "Output is different from reference file (gold): " ~ gold.escapePath;
156     }
157 
158     return res;
159 }
160 
161 struct CompareResult {
162     import std.ascii : newline;
163     import std.format : FormatSpec;
164 
165     // true if the golden file and result are _equal_.
166     bool status;
167 
168     string errorMsg;
169     string[] msg;
170     string[] lineDiff;
171 
172     void toString(Writer, Char)(scope Writer w, FormatSpec!Char fmt) const {
173         import std.algorithm : each;
174         import std.format : formattedWrite;
175         import std.range.primitives : put;
176 
177         formattedWrite(w, "status: %s\n", status);
178 
179         if (errorMsg.length) {
180             put(w, errorMsg);
181             put(w, newline);
182         }
183 
184         this.msg.each!((a) { put(w, a); put(w, newline); });
185         this.lineDiff.each!((a) { put(w, a); put(w, newline); });
186     }
187 
188     string toString() @safe pure const {
189         import std.exception : assumeUnique;
190         import std.format : FormatSpec;
191 
192         char[] buf;
193         buf.reserve(100);
194         auto fmt = FormatSpec!char("%s");
195         toString((const(char)[] s) { buf ~= s; }, fmt);
196         auto trustedUnique(T)(T t) @trusted {
197             return assumeUnique(t);
198         }
199 
200         return trustedUnique(buf);
201     }
202 }
203 
204 private auto nextFreeLogfile(string outdir) {
205     import std.file : exists;
206     import std.path : baseName;
207     import std..string : format;
208 
209     int idx;
210     string f;
211     do {
212         f = buildPath(outdir, format("run_compare%s.log", idx));
213         ++idx;
214     }
215     while (exists(f));
216 
217     return f;
218 }