devtool.tok_main source code

1 /**
2 Date: 2015-2017, Joakim Brännström
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module devtool.tok_main;
7 
8 import std.conv;
9 import std.stdio;
10 import std..string;
11 import std.typecons : Yes;
12 import logger = std.experimental.logger;
13 
14 import clang.c.Index;
15 
16 import clang.Util;
17 
18 void showClangVersion() {
19     CXString version_ = clang_getClangVersion();
20     writef("%s\n", toD(version_));
21 }
22 
23 void showAllTokens(ref CXTranslationUnit tu, CXToken* tokens, uint numTokens) {
24     static auto _getTokenKindSpelling(CXTokenKind kind) {
25         with (CXTokenKind) switch (kind) {
26         case punctuation:
27             return "Punctuation";
28         case keyword:
29             return "Keyword";
30         case identifier:
31             return "Identifier";
32         case literal:
33             return "Literal";
34         case comment:
35             return "Comment";
36         default:
37             return "Unknown";
38         }
39     }
40 
41     writeln("=== show tokens ===");
42     writef("NumTokens: %d\n", numTokens);
43     for (auto i = 0U; i < numTokens; i++) {
44         CXToken* token = &tokens[i];
45         CXTokenKind kind = clang_getTokenKind(*token);
46         CXString spell = clang_getTokenSpelling(tu, *token);
47         CXSourceLocation loc = clang_getTokenLocation(tu, *token);
48 
49         CXFile file;
50         uint line, column, offset;
51         clang_getFileLocation(loc, &file, &line, &column, &offset);
52         CXString fileName = clang_getFileName(file);
53 
54         writef("Token: %d\n", i);
55         writef(" Text: %s\n", toD(spell));
56         writef(" Kind: %s\n", _getTokenKindSpelling(kind));
57         writef(" Location: %s:%d:%d:%d\n", toD(fileName), line, column, offset);
58         writef("\n");
59 
60         clang_disposeString(fileName);
61     }
62 }
63 
64 CXSourceRange get_filerange(ref CXTranslationUnit tu, in string filename) {
65     auto getFileSize(in string fileName) {
66         import std.file;
67 
68         return getSize(fileName);
69     }
70 
71     CXFile file = clang_getFile(tu, filename.toStringz);
72     uint fileSize = cast(uint) getFileSize(filename);
73 
74     // get top/last location of the file
75     CXSourceLocation topLoc = clang_getLocationForOffset(tu, file, 0);
76     CXSourceLocation lastLoc = clang_getLocationForOffset(tu, file, fileSize);
77     if (clang_equalLocations(topLoc, clang_getNullLocation())
78             || clang_equalLocations(lastLoc, clang_getNullLocation())) {
79         writef("cannot retrieve location\n");
80         throw new Exception("location");
81     }
82 
83     // make a range from locations
84     CXSourceRange range = clang_getRange(topLoc, lastLoc);
85     if (clang_Range_isNull(range)) {
86         writef("cannot retrieve range\n");
87         throw new Exception("range");
88     }
89 
90     return range;
91 }
92 
93 int tokenize(string filename) {
94     // create index w/ excludeDeclsFromPCH = 1, displayDiagnostics=1.
95     CXIndex index = clang_createIndex(1, 1);
96 
97     // create Translation Unit
98     CXTranslationUnit tu = clang_parseTranslationUnit(index,
99             filename.toStringz, null, 0, null, 0, 0);
100     if (tu == null) {
101         writef("Cannot parse translation unit\n");
102         return 1;
103     }
104 
105     // get CXSouceRange of the file
106     CXSourceRange range = get_filerange(tu, filename);
107 
108     // tokenize in the range
109     CXToken* tokens;
110     uint numTokens;
111     clang_tokenize(tu, range, &tokens, &numTokens);
112 
113     // show tokens
114     showAllTokens(tu, tokens, numTokens);
115 
116     clang_disposeTokens(tu, tokens, numTokens);
117     clang_disposeTranslationUnit(tu);
118     clang_disposeIndex(index);
119 
120     return 0;
121 }
122 
123 int dumpAst(string filename, string[] flags) {
124     import cpptooling.analyzer.clang.context;
125     import clang.TranslationUnit : dumpAST;
126 
127     auto ctx = ClangContext(Yes.useInternalHeaders, Yes.prependParamSyntaxOnly);
128     auto tu = ctx.makeTranslationUnit(filename, flags);
129     writeln(dumpAST(tu));
130 
131     return 0;
132 }
133 
134 int dumpIncludes(string filename, string[] flags) {
135     import std.ascii;
136     import std.algorithm : filter;
137     import cpptooling.analyzer.clang.context;
138     import cpptooling.analyzer.clang.include_visitor;
139     import cpptooling.analyzer.clang.cursor_visitor;
140 
141     auto ctx = ClangContext(Yes.useInternalHeaders, Yes.prependParamSyntaxOnly);
142     auto tu = ctx.makeTranslationUnit(filename, flags);
143 
144     foreach (c; tu.cursor.visitBreathFirst.filter!(a => a.kind == CXCursorKind.inclusionDirective)) {
145         writefln(`#include "%s"%s  %s`, c.spelling, newline, c.include.file);
146     }
147 
148     return 0;
149 }
150 
151 int dumpBody(string fname, string[] flags) {
152     import std.typecons : scoped, NullableRef;
153     import cpptooling.analyzer.clang.analyze_helper;
154     import cpptooling.analyzer.clang.ast;
155     import cpptooling.analyzer.clang.context;
156     import cpptooling.analyzer.clang.check_parse_result : hasParseErrors,
157         logDiagnostic;
158     import cpptooling.analyzer.clang.cursor_logger : logNode, mixinNodeLog;
159     import cpptooling.data.symbol : Container;
160 
161     final class BodyVisitor : Visitor {
162         alias visit = Visitor.visit;
163         mixin generateIndentIncrDecr;
164 
165         NullableRef!Container container;
166 
167         this(ref Container container) {
168             this.container = &container;
169         }
170 
171         override void visit(const(Declaration) v) {
172             mixin(mixinNodeLog!());
173             v.accept(this);
174         }
175 
176         override void visit(const(Statement) v) {
177             mixin(mixinNodeLog!());
178             v.accept(this);
179         }
180 
181         override void visit(const(Expression) v) {
182             mixin(mixinNodeLog!());
183             v.accept(this);
184         }
185 
186         override void visit(const(CallExpr) v) {
187             mixin(mixinNodeLog!());
188 
189             auto c_func = v.cursor.referenced;
190             logger.tracef("ref:'%s' kind:%s", c_func.spelling, c_func.kind.to!string());
191 
192             if (c_func.kind == CXCursorKind.functionDecl) {
193                 auto result = analyzeFunctionDecl(c_func, container, indent);
194                 () @trusted{ logger.trace("result: ", result); }();
195             } else if (c_func.kind == CXCursorKind.cxxMethod) {
196                 auto result = analyzeCxxMethod(c_func, container, indent);
197                 () @trusted{ logger.trace("result: ", result); }();
198             } else {
199                 logger.trace("unknown callexpr: ", c_func.kind.to!string());
200             }
201 
202             v.accept(this);
203         }
204 
205         override void visit(const(DeclRefExpr) v) {
206             mixin(mixinNodeLog!());
207 
208             auto c_ref = v.cursor.referenced;
209             logger.trace("ref: ", c_ref.spelling, " kind:", c_ref.kind.to!string());
210 
211             v.accept(this);
212         }
213 
214         override void visit(const(MemberRefExpr) v) {
215             mixin(mixinNodeLog!());
216 
217             auto c_ref = v.cursor.referenced;
218             logger.trace("ref: ", c_ref.spelling, " kind:", c_ref.kind.to!string());
219 
220             v.accept(this);
221         }
222     }
223 
224     final class MainVisitor : Visitor {
225         alias visit = Visitor.visit;
226         mixin generateIndentIncrDecr;
227 
228         Container container;
229 
230         override void visit(const(TranslationUnit) v) {
231             mixin(mixinNodeLog!());
232             v.accept(this);
233         }
234 
235         override void visit(const(ClassDecl) v) {
236             mixin(mixinNodeLog!());
237             v.accept(this);
238         }
239 
240         override void visit(const(StructDecl) v) {
241             mixin(mixinNodeLog!());
242             v.accept(this);
243         }
244 
245         override void visit(const(FunctionDecl) v) @trusted {
246             mixin(mixinNodeLog!());
247 
248             auto visitor = scoped!(BodyVisitor)(container);
249             visitor.indent = indent;
250             v.accept(visitor);
251         }
252 
253         override void visit(const(CxxMethod) v) @trusted {
254             mixin(mixinNodeLog!());
255 
256             auto visitor = scoped!(BodyVisitor)(container);
257             visitor.indent = indent;
258             v.accept(visitor);
259         }
260     }
261 
262     auto ctx = ClangContext(Yes.useInternalHeaders, Yes.prependParamSyntaxOnly);
263     auto tu = ctx.makeTranslationUnit(fname, flags);
264 
265     if (tu.hasParseErrors) {
266         logDiagnostic(tu);
267     }
268 
269     auto visitor = new MainVisitor;
270     auto ast = ClangAST!(typeof(visitor))(tu.cursor);
271     ast.accept(visitor);
272 
273     return 0;
274 }
275 
276 int main(string[] args) {
277     import dextool.logger_conf : confLogLevel, ConfigureLog;
278 
279     confLogLevel(ConfigureLog.debug_);
280 
281     if (args.length < 3) {
282         writeln("devtool <category> filename");
283         writeln("categories: tok, dumpast, dumpbody, includes");
284         return 1;
285     }
286 
287     string[] flags;
288     if (args.length > 3) {
289         flags = args[3 .. $];
290     }
291 
292     showClangVersion();
293 
294     switch (args[1]) {
295     case "tok":
296         return tokenize(args[2]);
297     case "dumpast":
298         return dumpAst(args[2], flags);
299     case "dumpbody":
300         return dumpBody(args[2], flags);
301     case "includes":
302         return dumpIncludes(args[2], flags);
303     default:
304         return 1;
305     }
306 }