1 /**
2  The main entry point for the reggae tool. Its tasks are:
3  $(UL
4    $(LI Verify that a $(D reggafile.d) exists in the selected directory)
5    $(LI Generate a $(D reggaefile.d) for dub projects)
6    $(LI Write out the reggae library files and $(D config.d))
7    $(LI Compile the build description with the reggae library files to produce $(D buildgen))
8    $(LI Produce $(D dcompile), a binary to call the D compiler to obtain dependencies during compilation)
9    $(LI Call the produced $(D buildgen) binary)
10  )
11  */
12 
13 module reggae.reggae;
14 
15 import std.stdio;
16 import std.process: execute, environment;
17 import std.array: array, join, empty, split;
18 import std.typetuple;
19 import std.file;
20 import std.conv: text;
21 import std.exception: enforce;
22 import std.conv: to;
23 import std.algorithm;
24 
25 import reggae.options;
26 import reggae.ctaa;
27 import reggae.types;
28 import reggae.file;
29 import reggae.path: buildPath;
30 
31 
32 version(minimal) {
33     //empty stubs for minimal version of reggae
34     void maybeCreateReggaefile(T...)(T) {}
35     void writeDubConfig(T...)(T) {}
36 } else {
37     import reggae.dub.interop: writeDubConfig, maybeCreateReggaefile;
38 }
39 
40 mixin template reggaeGen(targets...) {
41     mixin buildImpl!targets;
42     mixin ReggaeMain;
43 }
44 
45 mixin template ReggaeMain() {
46     import reggae.options: getOptions;
47     import std.stdio: stdout, stderr;
48 
49     int main(string[] args) {
50         try {
51             run(stdout, args);
52         } catch(Exception ex) {
53             stderr.writeln(ex.msg);
54             return 1;
55         }
56 
57         return 0;
58     }
59 }
60 
61 void run(T)(auto ref T output, string[] args) {
62     auto options = getOptions(args);
63     run(output, options);
64 }
65 
66 void run(T)(auto ref T output, Options options) {
67 
68     if(options.earlyExit) return;
69 
70     enforce(options.projectPath != "", "A project path must be specified");
71 
72     // write out the library source files to be compiled/interpreted
73     // with the user's build description
74     writeSrcFiles(output, options);
75 
76     if(options.isJsonBuild) {
77         immutable haveToReturn = jsonBuild(options);
78         if(haveToReturn) return;
79     }
80 
81     maybeCreateReggaefile(output, options);
82     createBuild(output, options);
83 }
84 
85 //get JSON description of the build from a scripting language
86 //and transform it into a build description
87 //return true if no D files are present
88 bool jsonBuild(Options options) {
89     immutable jsonOutput = getJsonOutput(options);
90     return jsonBuild(options, jsonOutput);
91 }
92 
93 //transform JSON description into a Build struct
94 //return true if no D files are present
95 bool jsonBuild(Options options, in string jsonOutput) {
96     enforce(options.backend != Backend.binary, "Binary backend not supported via JSON");
97 
98     version(minimal)
99         assert(0, "JSON builds not supported in minimal version");
100     else {
101         import reggae.json_build;
102         import reggae.buildgen;
103         import reggae.rules.common: Language;
104 
105         auto build = jsonToBuild(options.projectPath, jsonOutput);
106         doBuild(build, jsonToOptions(options, jsonOutput));
107 
108         import reggae.buildgen:writeCompilationDB;
109         if(!options.noCompilationDB) writeCompilationDB(build, options);
110 
111         //true -> exit early
112         return !build.targets.canFind!(a => a.getLanguage == Language.D);
113     }
114 }
115 
116 
117 private string getJsonOutput(in Options options) @safe {
118     const args = getJsonOutputArgs(options);
119     const path = environment.get("PATH", "").split(":");
120     const pythonPaths = environment.get("PYTHONPATH", "").split(":");
121     const nodePaths = environment.get("NODE_PATH", "").split(":");
122     const luaPaths = environment.get("LUA_PATH", "").split(";");
123     const srcDir = buildPath(options.workingDir, hiddenDir, "src");
124     const binDir = buildPath(srcDir, "reggae");
125     auto env = ["PATH": (path ~ binDir).join(":"),
126                 "PYTHONPATH": (pythonPaths ~ srcDir).join(":"),
127                 "NODE_PATH": (nodePaths ~ options.projectPath ~ binDir).join(":"),
128                 "LUA_PATH": (luaPaths ~ buildPath(options.projectPath, "?.lua") ~ buildPath(binDir, "?.lua")).join(";")];
129     immutable res = execute(args, env);
130     enforce(res.status == 0, text("Could not execute ", args.join(" "), ":\n", res.output));
131     return res.output;
132 }
133 
134 private string[] getJsonOutputArgs(in Options options) @safe {
135 
136     import std.process: environment;
137     import std.json: parseJSON;
138 
139     final switch(options.reggaeFileLanguage) {
140 
141     case BuildLanguage.D:
142         assert(0, "Cannot obtain JSON build for builds written in D");
143 
144     case BuildLanguage.Python:
145 
146         auto optionsString = () @trusted {
147             import std.json;
148             import std.traits;
149             auto jsonVal = parseJSON(`{}`);
150             foreach(member; __traits(allMembers, typeof(options))) {
151                 static if(is(typeof(mixin(`options.` ~ member)) == const(Backend)) ||
152                           is(typeof(mixin(`options.` ~ member)) == const(string)) ||
153                           is(typeof(mixin(`options.` ~ member)) == const(bool)) ||
154                           is(typeof(mixin(`options.` ~ member)) == const(string[string])) ||
155                           is(typeof(mixin(`options.` ~ member)) == const(string[])))
156                     jsonVal.object[member] = mixin(`options.` ~ member);
157             }
158             return jsonVal.toString;
159         }();
160 
161         const haveReggaePython = "REGGAE_PYTHON" in environment;
162         auto pythonParts = haveReggaePython
163             ? [environment["REGGAE_PYTHON"]]
164             : ["/usr/bin/env", "python"];
165         return pythonParts ~ ["-B", "-m", "reggae.reggae_json_build",
166                 "--options", optionsString,
167                 options.projectPath];
168 
169     case BuildLanguage.Ruby:
170         return ["ruby", "-S",
171                 "-I" ~ options.projectPath,
172                 "-I" ~ buildPath(options.workingDir, hiddenDir, "src/reggae"),
173                 "reggae_json_build.rb"];
174 
175     case BuildLanguage.Lua:
176         return ["lua", buildPath(options.workingDir, hiddenDir, "src/reggae/reggae_json_build.lua")];
177 
178     case BuildLanguage.JavaScript:
179         return ["node", buildPath(options.workingDir, hiddenDir, "src/reggae/reggae_json_build.js")];
180     }
181 }
182 
183 enum coreFiles = [
184     "options.d",
185     "buildgen_main.d", "buildgen.d",
186     "build.d",
187     "backend/package.d", "backend/binary.d",
188     "package.d", "range.d", "reflect.d",
189     "dependencies.d", "types.d", "dcompile.d",
190     "ctaa.d", "sorting.d", "file.d",
191     "rules/package.d",
192     "rules/common.d",
193     "rules/d.d",
194     "rules/c_and_cpp.d",
195     "core/package.d", "core/rules/package.d",
196     ];
197 enum otherFiles = [
198     "backend/ninja.d", "backend/make.d", "backend/tup.d",
199     "dub/info.d", "rules/dub.d",
200     "path.d",
201     ];
202 
203 version(minimal) {
204     enum string[] foreignFiles = [];
205 } else {
206     enum foreignFiles = [
207         "__init__.py", "build.py", "reflect.py", "rules.py", "reggae_json_build.py",
208         "reggae.rb", "reggae_json_build.rb",
209         "reggae-js.js", "reggae_json_build.js",
210         "JSON.lua", "reggae.lua", "reggae_json_build.lua",
211         ];
212 }
213 
214 //all files that need to be written out and compiled
215 private string[] fileNames() @safe pure nothrow {
216     version(minimal)
217         return coreFiles;
218     else
219         return coreFiles ~ otherFiles;
220 }
221 
222 
223 private void createBuild(T)(auto ref T output, in Options options) {
224 
225     import reggae.io: log;
226 
227     enforce(options.reggaeFilePath.exists, text("Could not find ", options.reggaeFilePath));
228 
229     //compile the binaries (the build generator and dcompile)
230     immutable buildGenName = compileBinaries(output, options);
231 
232     //binary backend has no build generator, it _is_ the build
233     if(options.backend == Backend.binary) return;
234 
235     //only got here to build .dcompile
236     if(options.isScriptBuild) return;
237 
238     //actually run the build generator
239     output.log("Running the created binary to generate the build");
240     immutable retRunBuildgen = execute([buildPath(options.workingDir, hiddenDir, buildGenName)]);
241     enforce(retRunBuildgen.status == 0,
242             text("Couldn't execute the produced ", buildGenName, " binary:\n", retRunBuildgen.output));
243     output.log("Build generated");
244 
245     if(retRunBuildgen.output.length) output.log(retRunBuildgen.output);
246 }
247 
248 
249 struct Binary {
250     string name;
251     const(string)[] cmd;
252 }
253 
254 
255 private string compileBinaries(T)(auto ref T output, in Options options) {
256 
257     import reggae.rules.common: exeExt, objExt;
258 
259     buildDCompile(output, options);
260 
261     immutable buildGenName = getBuildGenName(options) ~ exeExt;
262     if(options.isScriptBuild) return buildGenName;
263 
264     const buildGenCmd = getCompileBuildGenCmd(options);
265     immutable buildObjName = "build" ~ objExt;
266     buildBinary(output, options, Binary(buildObjName, buildGenCmd));
267 
268     const reggaeFileDeps = getReggaeFileDependenciesDlang;
269     auto objFiles = [buildObjName];
270     if(!reggaeFileDeps.empty) {
271         immutable rest = "rest" ~ objExt;
272         buildBinary(output,
273                     options,
274                     Binary(rest,
275                            [options.dCompiler,
276                             "-c",
277                             "-of" ~ rest] ~
278                            importPaths(options) ~
279                            reggaeFileDeps));
280         objFiles ~= rest;
281     }
282 
283     buildBinary(output,
284                 options,
285                 Binary(buildGenName,
286                        [options.dCompiler, "-of" ~ buildGenName] ~ objFiles));
287 
288     return buildGenName;
289 }
290 
291 void buildDCompile(T)(auto ref T output, in Options options) {
292     import reggae.rules.common : exeExt;
293 
294     enum dcompileExe = "dcompile" ~ exeExt;
295 
296     if(!thisExePath.newerThan(buildPath(options.workingDir, hiddenDir, dcompileExe)))
297         return;
298 
299     immutable cmd = [options.dCompiler,
300                      "-Isrc",
301                      "-of" ~ dcompileExe,
302                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dcompile.d"),
303                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dependencies.d")];
304 
305     buildBinary(output, options, Binary(dcompileExe, cmd));
306 }
307 
308 private bool isExecutable(in char[] path) @trusted nothrow //TODO: @safe
309 {
310     version(Posix) {
311         import core.sys.posix.unistd;
312         import std.internal.cstring;
313         return (access(path.tempCString(), X_OK) == 0);
314     } else {
315         import core.sys.windows.winbase: GetBinaryTypeW;
316         import core.sys.windows.windef: DWORD;
317         import std.conv: to;
318 
319         DWORD type;
320         try
321             return GetBinaryTypeW(&path.to!wstring[0], &type) != 0;
322         catch(Exception _)
323             assert(false, "Conversion erro from string to wstring");
324     }
325 }
326 
327 private void buildBinary(T)(auto ref T output, in Options options, in Binary bin) {
328     import reggae.io: log;
329     import std.process;
330 
331     string[string] env;
332     auto config = Config.none;
333     auto maxOutput = size_t.max;
334     auto workDir = buildPath(options.workingDir, hiddenDir);
335     const extraInfo = options.verbose ? " with " ~  bin.cmd.join(" ") : "";
336     output.log("Compiling metabuild binary ", bin.name, extraInfo);
337     // std.process.execute has a bug where using workDir and a relative path
338     // don't work (https://issues.dlang.org/show_bug.cgi?id=15915)
339     // so executeShell is used instead
340     immutable res = executeShell(bin.cmd.join(" "), env, config, maxOutput, workDir);
341     enforce(res.status == 0, text("Couldn't execute ", bin.cmd.join(" "), "\nin ", workDir,
342                                   ":\n", res.output,
343                                   "\n", "bin.name: ", bin.name, ", bin.cmd: ", bin.cmd.join(" ")));
344 
345 }
346 
347 
348 private const(string)[] getCompileBuildGenCmd(in Options options) @safe {
349     import reggae.rules.common: objExt;
350 
351     const reggaeSrcs = ("config.d" ~ fileNames).
352         filter!(a => a != "dcompile.d").
353         map!(a => buildPath(reggaeSrcRelDirName, a)).array;
354 
355     immutable buildBinFlags = options.backend == Backend.binary
356         ? ["-O", "-inline"]
357         : [];
358     enum dcompile = buildPath("./dcompile");
359     const commonBefore = [dcompile,
360                           "--objFile=" ~ "build" ~ objExt,
361                           "--depFile=" ~ "reggaefile.dep",
362                           options.dCompiler] ~
363         importPaths(options)
364         // ~ ["-g", "-debug"]
365         ;
366     const commonAfter = buildBinFlags ~ options.reggaeFilePath ~ reggaeSrcs;
367     version(minimal) return commonBefore ~ "-version=minimal" ~ commonAfter;
368     else return commonBefore ~ commonAfter;
369 }
370 
371 private string[] importPaths(in Options options) @safe nothrow {
372     import std.file;
373 
374     immutable srcDir = "-I" ~ buildPath("src");
375     // if compiling phobos, the includes for the reggaefile.d compilation
376     // will pick up the new phobos if we include the src path
377     return "std".exists ? [srcDir] : ["-I" ~ options.projectPath, srcDir];
378 }
379 
380 private string getBuildGenName(in Options options) @safe pure nothrow {
381     return options.backend == Backend.binary ? buildPath("../build") : "buildgen";
382 }
383 
384 
385 immutable reggaeSrcRelDirName = buildPath("src/reggae");
386 
387 string reggaeSrcDirName(in Options options) @safe pure nothrow {
388     return buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName);
389 }
390 
391 
392 void writeSrcFiles(T)(auto ref T output, in Options options) {
393     import reggae.io: log;
394 
395     output.log("Writing reggae source files");
396 
397     import std.file: mkdirRecurse;
398     immutable reggaeSrcDirName = reggaeSrcDirName(options);
399     if(!reggaeSrcDirName.exists) {
400         mkdirRecurse(reggaeSrcDirName);
401         mkdirRecurse(buildPath(reggaeSrcDirName, "dub"));
402         mkdirRecurse(buildPath(reggaeSrcDirName, "rules"));
403         mkdirRecurse(buildPath(reggaeSrcDirName, "backend"));
404         mkdirRecurse(buildPath(reggaeSrcDirName, "core/rules"));
405     }
406 
407     //this foreach has to happen at compile time due
408     //to the string import below.
409     foreach(fileName; aliasSeqOf!(fileNames ~ foreignFiles)) {
410         auto file = File(reggaeSrcFileName(options, fileName), "w");
411         file.write(import(fileName));
412     }
413 
414     writeConfig(output, options);
415 }
416 
417 
418 private void writeConfig(T)(auto ref T output, in Options options) {
419 
420     import reggae.io: log;
421 
422     output.log("Writing reggae configuration");
423 
424     auto file = File(reggaeSrcFileName(options, "config.d"), "w");
425 
426     file.writeln(q{
427 module reggae.config;
428 import reggae.ctaa;
429 import reggae.types;
430 import reggae.options;
431     });
432 
433     version(minimal) file.writeln("enum isDubProject = false;");
434     file.writeln("immutable options = ", options, ";");
435 
436     file.writeln("enum userVars = AssocList!(string, string)([");
437     foreach(key, value; options.userVars) {
438         file.writeln("assocEntry(`", key, "`, `", value, "`), ");
439     }
440     file.writeln("]);");
441 
442     try {
443         writeDubConfig(output, options, file);
444     } catch(Exception ex) {
445         stderr.writeln("Could not write dub configuration: ", ex.msg);
446         throw ex;
447     }
448 }
449 
450 
451 private string reggaeSrcFileName(in Options options, in string fileName) @safe pure nothrow {
452     return buildPath(reggaeSrcDirName(options), fileName);
453 }