1 /**
2  The main entry point for the reggae tool. Its tasks are:
3  $(UL
4    $(LI Verify that a $(D reggafile.d) exists in the selected directory)
5    $(LI Generate a $(D reggaefile.d) for dub projects)
6    $(LI Write out the reggae library files and $(D config.d))
7    $(LI Compile the build description with the reggae library files to produce $(D buildgen))
8    $(LI Produce $(D dcompile), a binary to call the D compiler to obtain dependencies during compilation)
9    $(LI Call the produced $(D buildgen) binary)
10  )
11  */
12 
13 module reggae.reggae;
14 
15 import std.stdio;
16 import std.process: execute, environment;
17 import std.array: array, join, empty, split;
18 import std.path: absolutePath, buildPath, relativePath;
19 import std.typetuple;
20 import std.file;
21 import std.conv: text;
22 import std.exception: enforce;
23 import std.conv: to;
24 import std.algorithm;
25 
26 import reggae.options;
27 import reggae.ctaa;
28 import reggae.types;
29 import reggae.file;
30 
31 
32 version(minimal) {
33     //empty stubs for minimal version of reggae
34     void maybeCreateReggaefile(T...)(T) {}
35     void writeDubConfig(T...)(T) {}
36 } else {
37     import reggae.dub.interop;
38 }
39 
40 mixin template reggaeGen(targets...) {
41     mixin buildImpl!targets;
42     mixin ReggaeMain;
43 }
44 
45 mixin template ReggaeMain() {
46     import reggae.options: getOptions;
47     import std.stdio: stdout, stderr;
48 
49     int main(string[] args) {
50         try {
51             run(stdout, args);
52         } catch(Exception ex) {
53             stderr.writeln(ex.msg);
54             return 1;
55         }
56 
57         return 0;
58     }
59 }
60 
61 void run(T)(auto ref T output, string[] args) {
62     auto options = getOptions(args);
63     run(output, options);
64 }
65 
66 void run(T)(auto ref T output, Options options) {
67     if(options.earlyExit) return;
68     enforce(options.projectPath != "", "A project path must be specified");
69 
70     // write out the library source files to be compiled/interpreted
71     // with the user's build description
72     writeSrcFiles(output, options);
73 
74     if(options.isJsonBuild) {
75         immutable haveToReturn = jsonBuild(options);
76         if(haveToReturn) return;
77     }
78 
79     maybeCreateReggaefile(output, options);
80     createBuild(output, options);
81 }
82 
83 //get JSON description of the build from a scripting language
84 //and transform it into a build description
85 //return true if no D files are present
86 bool jsonBuild(Options options) {
87     immutable jsonOutput = getJsonOutput(options);
88     return jsonBuild(options, jsonOutput);
89 }
90 
91 //transform JSON description into a Build struct
92 //return true if no D files are present
93 bool jsonBuild(Options options, in string jsonOutput) {
94     enforce(options.backend != Backend.binary, "Binary backend not supported via JSON");
95 
96     version(minimal)
97         assert(0, "JSON builds not supported in minimal version");
98     else {
99         import reggae.json_build;
100         import reggae.buildgen;
101         import reggae.rules.common: Language;
102 
103         auto build = jsonToBuild(options.projectPath, jsonOutput);
104         doBuild(build, jsonToOptions(options, jsonOutput));
105 
106         import reggae.buildgen:writeCompilationDB;
107         if(!options.noCompilationDB) writeCompilationDB(build, options);
108 
109         //true -> exit early
110         return !build.targets.canFind!(a => a.getLanguage == Language.D);
111     }
112 }
113 
114 
115 private string getJsonOutput(in Options options) @safe {
116     const args = getJsonOutputArgs(options);
117     const path = environment.get("PATH", "").split(":");
118     const pythonPaths = environment.get("PYTHONPATH", "").split(":");
119     const nodePaths = environment.get("NODE_PATH", "").split(":");
120     const luaPaths = environment.get("LUA_PATH", "").split(";");
121     const srcDir = buildPath(options.workingDir, hiddenDir, "src");
122     const binDir = buildPath(srcDir, "reggae");
123     auto env = ["PATH": (path ~ binDir).join(":"),
124                 "PYTHONPATH": (pythonPaths ~ srcDir).join(":"),
125                 "NODE_PATH": (nodePaths ~ options.projectPath ~ binDir).join(":"),
126                 "LUA_PATH": (luaPaths ~ buildPath(options.projectPath, "?.lua") ~ buildPath(binDir, "?.lua")).join(";")];
127     immutable res = execute(args, env);
128     enforce(res.status == 0, text("Could not execute ", args.join(" "), ":\n", res.output));
129     return res.output;
130 }
131 
132 private string[] getJsonOutputArgs(in Options options) @safe {
133     final switch(options.reggaeFileLanguage) {
134 
135     case BuildLanguage.D:
136         assert(0, "Cannot obtain JSON build for builds written in D");
137 
138     case BuildLanguage.Python:
139 
140         auto optionsString = () @trusted {
141             import std.json;
142             import std.traits;
143             JSONValue jsonVal = parseJSON(`{}`);
144             foreach(member; __traits(allMembers, typeof(options))) {
145                 static if(is(typeof(mixin(`options.` ~ member)) == const(Backend)) ||
146                           is(typeof(mixin(`options.` ~ member)) == const(string)) ||
147                           is(typeof(mixin(`options.` ~ member)) == const(bool)) ||
148                           is(typeof(mixin(`options.` ~ member)) == const(string[string])) ||
149                           is(typeof(mixin(`options.` ~ member)) == const(string[])))
150                     jsonVal.object[member] = mixin(`options.` ~ member);
151             }
152             return jsonVal.toString;
153         }();
154 
155         return ["/usr/bin/env", "python", "-m", "reggae.reggae_json_build",
156                 "--options", optionsString,
157                 options.projectPath];
158 
159     case BuildLanguage.Ruby:
160         return ["ruby", "-S",
161                 "-I" ~ options.projectPath,
162                 "-I" ~ buildPath(options.workingDir, hiddenDir, "src", "reggae"),
163                 "reggae_json_build.rb"];
164 
165     case BuildLanguage.Lua:
166         return ["lua", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.lua")];
167 
168     case BuildLanguage.JavaScript:
169         return ["node", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.js")];
170     }
171 }
172 
173 enum coreFiles = [
174     "options.d",
175     "buildgen_main.d", "buildgen.d",
176     "build.d",
177     "backend/package.d", "backend/binary.d",
178     "package.d", "range.d", "reflect.d",
179     "dependencies.d", "types.d", "dcompile.d",
180     "ctaa.d", "sorting.d", "file.d",
181     "rules/package.d",
182     "rules/common.d",
183     "rules/d.d",
184     "rules/c_and_cpp.d",
185     "core/package.d", "core/rules/package.d",
186     ];
187 enum otherFiles = [
188     "backend/ninja.d", "backend/make.d", "backend/tup.d",
189     "dub/info.d", "rules/dub.d",
190     ];
191 
192 version(minimal) {
193     enum string[] foreignFiles = [];
194 } else {
195     enum foreignFiles = [
196         "__init__.py", "build.py", "reflect.py", "rules.py", "reggae_json_build.py",
197         "reggae.rb", "reggae_json_build.rb",
198         "reggae-js.js", "reggae_json_build.js",
199         "JSON.lua", "reggae.lua", "reggae_json_build.lua",
200         ];
201 }
202 
203 //all files that need to be written out and compiled
204 private string[] fileNames() @safe pure nothrow {
205     version(minimal)
206         return coreFiles;
207     else
208         return coreFiles ~ otherFiles;
209 }
210 
211 
212 private void createBuild(T)(auto ref T output, in Options options) {
213 
214     enforce(options.reggaeFilePath.exists, text("Could not find ", options.reggaeFilePath));
215 
216     //compile the binaries (the build generator and dcompile)
217     immutable buildGenName = compileBinaries(output, options);
218 
219     //binary backend has no build generator, it _is_ the build
220     if(options.backend == Backend.binary) return;
221 
222     //only got here to build .dcompile
223     if(options.isScriptBuild) return;
224 
225     //actually run the build generator
226     output.writeln("[Reggae] Running the created binary to generate the build");
227     immutable retRunBuildgen = execute([buildPath(options.workingDir, hiddenDir, buildGenName)]);
228     enforce(retRunBuildgen.status == 0,
229             text("Couldn't execute the produced ", buildGenName, " binary:\n", retRunBuildgen.output));
230 
231     output.writeln(retRunBuildgen.output);
232 }
233 
234 
235 struct Binary {
236     string name;
237     const(string)[] cmd;
238 }
239 
240 
241 private string compileBinaries(T)(auto ref T output, in Options options) {
242     buildDCompile(output, options);
243 
244     immutable buildGenName = getBuildGenName(options);
245     if(options.isScriptBuild) return buildGenName;
246 
247     const buildGenCmd = getCompileBuildGenCmd(options);
248     immutable buildObjName = "build.o";
249     buildBinary(output, options, Binary(buildObjName, buildGenCmd));
250 
251     const reggaeFileDeps = getReggaeFileDependenciesDlang;
252     auto objFiles = [buildObjName];
253     if(!reggaeFileDeps.empty) {
254         immutable rest = "rest.o";
255         buildBinary(output,
256                     options,
257                     Binary(rest,
258                            [options.dCompiler,
259                             "-c",
260                             "-of" ~ "rest.o"] ~
261                            importPaths(options) ~
262                            reggaeFileDeps));
263         objFiles ~= rest;
264     }
265     buildBinary(output, options, Binary(buildGenName, [options.dCompiler, "-of" ~ buildGenName] ~ objFiles));
266 
267     return buildGenName;
268 }
269 
270 void buildDCompile(T)(auto ref T output, in Options options) {
271     if(!thisExePath.newerThan(buildPath(options.workingDir, hiddenDir, "dcompile")))
272         return;
273 
274     immutable cmd = [options.dCompiler,
275                      "-Isrc",
276                      "-ofdcompile",
277                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dcompile.d"),
278                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dependencies.d")];
279 
280     buildBinary(output, options, Binary("dcompile", cmd));
281 }
282 
283 private bool isExecutable(in char[] path) @trusted nothrow @nogc //TODO: @safe
284 {
285     import core.sys.posix.unistd;
286     import std.internal.cstring;
287     return (access(path.tempCString(), X_OK) == 0);
288 }
289 
290 private void buildBinary(T)(auto ref T output, in Options options, in Binary bin) {
291     import std.process;
292     string[string] env;
293     auto config = Config.none;
294     auto maxOutput = size_t.max;
295     auto workDir = buildPath(options.workingDir, hiddenDir);
296     output.write("[Reggae] Compiling metabuild binary ", bin.name);
297     if(options.verbose) output.write(" with ", bin.cmd.join(" "));
298     output.writeln;
299     // std.process.execute has a bug where using workDir and a relative path
300     // don't work (https://issues.dlang.org/show_bug.cgi?id=15915)
301     // so executeShell is used instead
302     immutable res = executeShell(bin.cmd.join(" "), env, config, maxOutput, workDir);
303     enforce(res.status == 0, text("Couldn't execute ", bin.cmd.join(" "), "\nin ", workDir,
304                                   ":\n", res.output,
305                                   "\n", "bin.name: ", bin.name, ", bin.cmd: ", bin.cmd.join(" ")));
306 
307 }
308 
309 
310 private const(string)[] getCompileBuildGenCmd(in Options options) @safe {
311     import reggae.rules.common: objExt;
312 
313     const reggaeSrcs = ("config.d" ~ fileNames).
314         filter!(a => a != "dcompile.d").
315         map!(a => buildPath(reggaeSrcRelDirName, a)).array;
316 
317     immutable buildBinFlags = options.backend == Backend.binary
318         ? ["-O", "-inline"]
319         : [];
320     const commonBefore = ["./dcompile",
321                           "--objFile=" ~ "build.o",
322                           "--depFile=" ~ "reggaefile.dep",
323                           options.dCompiler] ~
324         importPaths(options) ~
325         ["-g",
326          "-debug"];
327     const commonAfter = buildBinFlags ~
328         options.reggaeFilePath ~ reggaeSrcs;
329     version(minimal) return commonBefore ~ "-version=minimal" ~ commonAfter;
330     else return commonBefore ~ commonAfter;
331 }
332 
333 private string[] importPaths(in Options options) @safe nothrow {
334     import std.file;
335 
336     immutable srcDir = "-I" ~ buildPath("src");
337     // if compiling phobos, the includes for the reggaefile.d compilation
338     // will pick up the new phobos if we include the src path
339     return "std".exists ? [srcDir] : ["-I" ~ options.projectPath, srcDir];
340 }
341 
342 private string getBuildGenName(in Options options) @safe pure nothrow {
343     return options.backend == Backend.binary ? buildPath("..", "build") : "buildgen";
344 }
345 
346 
347 immutable reggaeSrcRelDirName = buildPath("src", "reggae");
348 
349 string reggaeSrcDirName(in Options options) @safe pure nothrow {
350     return buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName);
351 }
352 
353 
354 void writeSrcFiles(T)(auto ref T output, in Options options) {
355     output.writeln("[Reggae] Writing reggae source files");
356 
357     import std.file: mkdirRecurse;
358     immutable reggaeSrcDirName = reggaeSrcDirName(options);
359     if(!reggaeSrcDirName.exists) {
360         mkdirRecurse(reggaeSrcDirName);
361         mkdirRecurse(buildPath(reggaeSrcDirName, "dub"));
362         mkdirRecurse(buildPath(reggaeSrcDirName, "rules"));
363         mkdirRecurse(buildPath(reggaeSrcDirName, "backend"));
364         mkdirRecurse(buildPath(reggaeSrcDirName, "core", "rules"));
365     }
366 
367     //this foreach has to happen at compile time due
368     //to the string import below.
369     foreach(fileName; aliasSeqOf!(fileNames ~ foreignFiles)) {
370         auto file = File(reggaeSrcFileName(options, fileName), "w");
371         file.write(import(fileName));
372     }
373 
374     output.writeln("[Reggae] Writing reggae configuration");
375     writeConfig(output, options);
376 }
377 
378 
379 private void writeConfig(T)(auto ref T output, in Options options) {
380     auto file = File(reggaeSrcFileName(options, "config.d"), "w");
381 
382     file.writeln(q{
383 module reggae.config;
384 import reggae.ctaa;
385 import reggae.types;
386 import reggae.options;
387     });
388 
389     version(minimal) file.writeln("enum isDubProject = false;");
390     file.writeln("immutable options = ", options, ";");
391 
392     file.writeln("enum userVars = AssocList!(string, string)([");
393     foreach(key, value; options.userVars) {
394         file.writeln("assocEntry(`", key, "`, `", value, "`), ");
395     }
396     file.writeln("]);");
397 
398     try {
399         writeDubConfig(output, options, file);
400     } catch(Exception ex) {
401         stderr.writeln("Could not write dub configuration, try 'dub upgrade': ", ex.msg);
402         throw ex;
403     }
404 }
405 
406 
407 private string reggaeSrcFileName(in Options options, in string fileName) @safe pure nothrow {
408     return buildPath(reggaeSrcDirName(options), fileName);
409 }