1 /**
2  The main entry point for the reggae tool. Its tasks are:
3  $(UL
4    $(LI Verify that a $(D reggafile.d) exists in the selected directory)
5    $(LI Generate a $(D reggaefile.d) for dub projects)
6    $(LI Write out the reggae library files and $(D config.d))
7    $(LI Compile the build description with the reggae library files to produce $(D buildgen))
8    $(LI Produce $(D dcompile), a binary to call the D compiler to obtain dependencies during compilation)
9    $(LI Call the produced $(D buildgen) binary)
10  )
11  */
12 
13 module reggae.reggae;
14 
15 import std.stdio;
16 import std.process: execute, environment;
17 import std.array: array, join, empty, split;
18 import std.path: absolutePath, buildPath, relativePath;
19 import std.typetuple;
20 import std.file;
21 import std.conv: text;
22 import std.exception: enforce;
23 import std.conv: to;
24 import std.algorithm;
25 
26 import reggae.options;
27 import reggae.ctaa;
28 import reggae.types;
29 import reggae.file;
30 
31 
32 version(minimal) {
33     //empty stubs for minimal version of reggae
34     void maybeCreateReggaefile(T...)(T) {}
35     void writeDubConfig(T...)(T) {}
36 } else {
37     import reggae.dub.interop;
38 }
39 
40 mixin template reggaeGen(targets...) {
41     mixin buildImpl!targets;
42     mixin ReggaeMain;
43 }
44 
45 mixin template ReggaeMain() {
46     import reggae.options: getOptions;
47     import std.stdio: stdout, stderr;
48 
49     int main(string[] args) {
50         try {
51             run(stdout, args);
52         } catch(Exception ex) {
53             stderr.writeln(ex.msg);
54             return 1;
55         }
56 
57         return 0;
58     }
59 }
60 
61 void run(T)(auto ref T output, string[] args) {
62     auto options = getOptions(args);
63     run(output, options);
64 }
65 
66 void run(T)(auto ref T output, Options options) {
67     if(options.earlyExit) return;
68     enforce(options.projectPath != "", "A project path must be specified");
69 
70     // write out the library source files to be compiled/interpreted
71     // with the user's build description
72     writeSrcFiles(output, options);
73 
74     if(options.isJsonBuild) {
75         immutable haveToReturn = jsonBuild(options);
76         if(haveToReturn) return;
77     }
78 
79     maybeCreateReggaefile(output, options);
80     createBuild(output, options);
81 }
82 
83 //get JSON description of the build from a scripting language
84 //and transform it into a build description
85 //return true if no D files are present
86 bool jsonBuild(Options options) {
87     immutable jsonOutput = getJsonOutput(options);
88     return jsonBuild(options, jsonOutput);
89 }
90 
91 //transform JSON description into a Build struct
92 //return true if no D files are present
93 bool jsonBuild(Options options, in string jsonOutput) {
94     enforce(options.backend != Backend.binary, "Binary backend not supported via JSON");
95 
96     version(minimal)
97         assert(0, "JSON builds not supported in minimal version");
98     else {
99         import reggae.json_build;
100         import reggae.buildgen;
101         import reggae.rules.common: Language;
102 
103         auto build = jsonToBuild(options.projectPath, jsonOutput);
104         doBuild(build, jsonToOptions(options, jsonOutput));
105 
106         import reggae.buildgen:writeCompilationDB;
107         if(!options.noCompilationDB) writeCompilationDB(build, options);
108 
109         //true -> exit early
110         return !build.targets.canFind!(a => a.getLanguage == Language.D);
111     }
112 }
113 
114 
115 private string getJsonOutput(in Options options) @safe {
116     const args = getJsonOutputArgs(options);
117     const path = environment.get("PATH", "").split(":");
118     const pythonPaths = environment.get("PYTHONPATH", "").split(":");
119     const nodePaths = environment.get("NODE_PATH", "").split(":");
120     const luaPaths = environment.get("LUA_PATH", "").split(";");
121     const srcDir = buildPath(options.workingDir, hiddenDir, "src");
122     const binDir = buildPath(srcDir, "reggae");
123     auto env = ["PATH": (path ~ binDir).join(":"),
124                 "PYTHONPATH": (pythonPaths ~ srcDir).join(":"),
125                 "NODE_PATH": (nodePaths ~ options.projectPath ~ binDir).join(":"),
126                 "LUA_PATH": (luaPaths ~ buildPath(options.projectPath, "?.lua") ~ buildPath(binDir, "?.lua")).join(";")];
127     immutable res = execute(args, env);
128     enforce(res.status == 0, text("Could not execute ", args.join(" "), ":\n", res.output));
129     return res.output;
130 }
131 
132 private string[] getJsonOutputArgs(in Options options) @safe {
133 
134     import std.process: environment;
135     import std.json: parseJSON;
136 
137     final switch(options.reggaeFileLanguage) {
138 
139     case BuildLanguage.D:
140         assert(0, "Cannot obtain JSON build for builds written in D");
141 
142     case BuildLanguage.Python:
143 
144         auto optionsString = () @trusted {
145             import std.json;
146             import std.traits;
147             auto jsonVal = parseJSON(`{}`);
148             foreach(member; __traits(allMembers, typeof(options))) {
149                 static if(is(typeof(mixin(`options.` ~ member)) == const(Backend)) ||
150                           is(typeof(mixin(`options.` ~ member)) == const(string)) ||
151                           is(typeof(mixin(`options.` ~ member)) == const(bool)) ||
152                           is(typeof(mixin(`options.` ~ member)) == const(string[string])) ||
153                           is(typeof(mixin(`options.` ~ member)) == const(string[])))
154                     jsonVal.object[member] = mixin(`options.` ~ member);
155             }
156             return jsonVal.toString;
157         }();
158 
159         const haveReggaePython = "REGGAE_PYTHON" in environment;
160         auto pythonParts = haveReggaePython
161             ? [environment["REGGAE_PYTHON"]]
162             : ["/usr/bin/env", "python"];
163         return pythonParts ~ ["-B", "-m", "reggae.reggae_json_build",
164                 "--options", optionsString,
165                 options.projectPath];
166 
167     case BuildLanguage.Ruby:
168         return ["ruby", "-S",
169                 "-I" ~ options.projectPath,
170                 "-I" ~ buildPath(options.workingDir, hiddenDir, "src", "reggae"),
171                 "reggae_json_build.rb"];
172 
173     case BuildLanguage.Lua:
174         return ["lua", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.lua")];
175 
176     case BuildLanguage.JavaScript:
177         return ["node", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.js")];
178     }
179 }
180 
181 enum coreFiles = [
182     "options.d",
183     "buildgen_main.d", "buildgen.d",
184     "build.d",
185     "backend/package.d", "backend/binary.d",
186     "package.d", "range.d", "reflect.d",
187     "dependencies.d", "types.d", "dcompile.d",
188     "ctaa.d", "sorting.d", "file.d",
189     "rules/package.d",
190     "rules/common.d",
191     "rules/d.d",
192     "rules/c_and_cpp.d",
193     "core/package.d", "core/rules/package.d",
194     ];
195 enum otherFiles = [
196     "backend/ninja.d", "backend/make.d", "backend/tup.d",
197     "dub/info.d", "rules/dub.d",
198     "path.d",
199     ];
200 
201 version(minimal) {
202     enum string[] foreignFiles = [];
203 } else {
204     enum foreignFiles = [
205         "__init__.py", "build.py", "reflect.py", "rules.py", "reggae_json_build.py",
206         "reggae.rb", "reggae_json_build.rb",
207         "reggae-js.js", "reggae_json_build.js",
208         "JSON.lua", "reggae.lua", "reggae_json_build.lua",
209         ];
210 }
211 
212 //all files that need to be written out and compiled
213 private string[] fileNames() @safe pure nothrow {
214     version(minimal)
215         return coreFiles;
216     else
217         return coreFiles ~ otherFiles;
218 }
219 
220 
221 private void createBuild(T)(auto ref T output, in Options options) {
222 
223     import reggae.io: log;
224 
225     enforce(options.reggaeFilePath.exists, text("Could not find ", options.reggaeFilePath));
226 
227     //compile the binaries (the build generator and dcompile)
228     immutable buildGenName = compileBinaries(output, options);
229 
230     //binary backend has no build generator, it _is_ the build
231     if(options.backend == Backend.binary) return;
232 
233     //only got here to build .dcompile
234     if(options.isScriptBuild) return;
235 
236     //actually run the build generator
237     output.log("Running the created binary to generate the build");
238     immutable retRunBuildgen = execute([buildPath(options.workingDir, hiddenDir, buildGenName)]);
239     enforce(retRunBuildgen.status == 0,
240             text("Couldn't execute the produced ", buildGenName, " binary:\n", retRunBuildgen.output));
241     output.log("Build generated");
242 
243     if(retRunBuildgen.output.length) output.log(retRunBuildgen.output);
244 }
245 
246 
247 struct Binary {
248     string name;
249     const(string)[] cmd;
250 }
251 
252 
253 private string compileBinaries(T)(auto ref T output, in Options options) {
254 
255     import reggae.rules.common: objExt;
256 
257     buildDCompile(output, options);
258 
259     immutable buildGenName = getBuildGenName(options);
260     if(options.isScriptBuild) return buildGenName;
261 
262     const buildGenCmd = getCompileBuildGenCmd(options);
263     immutable buildObjName = "build" ~ objExt;
264     buildBinary(output, options, Binary(buildObjName, buildGenCmd));
265 
266     const reggaeFileDeps = getReggaeFileDependenciesDlang;
267     auto objFiles = [buildObjName];
268     if(!reggaeFileDeps.empty) {
269         immutable rest = "rest" ~ objExt;
270         buildBinary(output,
271                     options,
272                     Binary(rest,
273                            [options.dCompiler,
274                             "-c",
275                             "-of" ~ rest] ~
276                            importPaths(options) ~
277                            reggaeFileDeps));
278         objFiles ~= rest;
279     }
280 
281     buildBinary(output,
282                 options,
283                 Binary(buildGenName,
284                        [options.dCompiler, "-of" ~ buildGenName] ~ objFiles));
285 
286     return buildGenName;
287 }
288 
289 void buildDCompile(T)(auto ref T output, in Options options) {
290     if(!thisExePath.newerThan(buildPath(options.workingDir, hiddenDir, "dcompile")))
291         return;
292 
293     immutable cmd = [options.dCompiler,
294                      "-Isrc",
295                      "-ofdcompile",
296                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dcompile.d"),
297                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dependencies.d")];
298 
299     buildBinary(output, options, Binary("dcompile", cmd));
300 }
301 
302 private bool isExecutable(in char[] path) @trusted nothrow //TODO: @safe
303 {
304     version(Posix) {
305         import core.sys.posix.unistd;
306         import std.internal.cstring;
307         return (access(path.tempCString(), X_OK) == 0);
308     } else {
309         import core.sys.windows.winbase: GetBinaryTypeW;
310         import core.sys.windows.windef: DWORD;
311         import std.conv: to;
312 
313         DWORD type;
314         try
315             return GetBinaryTypeW(&path.to!wstring[0], &type) != 0;
316         catch(Exception _)
317             assert(false, "Conversion erro from string to wstring");
318     }
319 }
320 
321 private void buildBinary(T)(auto ref T output, in Options options, in Binary bin) {
322     import reggae.io: log;
323     import std.process;
324 
325     string[string] env;
326     auto config = Config.none;
327     auto maxOutput = size_t.max;
328     auto workDir = buildPath(options.workingDir, hiddenDir);
329     const extraInfo = options.verbose ? " with " ~  bin.cmd.join(" ") : "";
330     output.log("Compiling metabuild binary ", bin.name, extraInfo);
331     // std.process.execute has a bug where using workDir and a relative path
332     // don't work (https://issues.dlang.org/show_bug.cgi?id=15915)
333     // so executeShell is used instead
334     immutable res = executeShell(bin.cmd.join(" "), env, config, maxOutput, workDir);
335     enforce(res.status == 0, text("Couldn't execute ", bin.cmd.join(" "), "\nin ", workDir,
336                                   ":\n", res.output,
337                                   "\n", "bin.name: ", bin.name, ", bin.cmd: ", bin.cmd.join(" ")));
338 
339 }
340 
341 
342 private const(string)[] getCompileBuildGenCmd(in Options options) @safe {
343     import reggae.rules.common: objExt;
344 
345     const reggaeSrcs = ("config.d" ~ fileNames).
346         filter!(a => a != "dcompile.d").
347         map!(a => buildPath(reggaeSrcRelDirName, a)).array;
348 
349     immutable buildBinFlags = options.backend == Backend.binary
350         ? ["-O", "-inline"]
351         : [];
352     version(Windows)
353         enum dcompile = "dcompile";
354     else
355         enum dcompile = "./dcompile";
356     const commonBefore = [dcompile,
357                           "--objFile=" ~ "build" ~ objExt,
358                           "--depFile=" ~ "reggaefile.dep",
359                           options.dCompiler] ~
360         importPaths(options) ~
361         ["-g",
362          "-debug"];
363     const commonAfter = buildBinFlags ~
364         options.reggaeFilePath ~ reggaeSrcs;
365     version(minimal) return commonBefore ~ "-version=minimal" ~ commonAfter;
366     else return commonBefore ~ commonAfter;
367 }
368 
369 private string[] importPaths(in Options options) @safe nothrow {
370     import std.file;
371 
372     immutable srcDir = "-I" ~ buildPath("src");
373     // if compiling phobos, the includes for the reggaefile.d compilation
374     // will pick up the new phobos if we include the src path
375     return "std".exists ? [srcDir] : ["-I" ~ options.projectPath, srcDir];
376 }
377 
378 private string getBuildGenName(in Options options) @safe pure nothrow {
379     return options.backend == Backend.binary ? buildPath("..", "build") : "buildgen";
380 }
381 
382 
383 immutable reggaeSrcRelDirName = buildPath("src", "reggae");
384 
385 string reggaeSrcDirName(in Options options) @safe pure nothrow {
386     return buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName);
387 }
388 
389 
390 void writeSrcFiles(T)(auto ref T output, in Options options) {
391     import reggae.io: log;
392 
393     output.log("Writing reggae source files");
394 
395     import std.file: mkdirRecurse;
396     immutable reggaeSrcDirName = reggaeSrcDirName(options);
397     if(!reggaeSrcDirName.exists) {
398         mkdirRecurse(reggaeSrcDirName);
399         mkdirRecurse(buildPath(reggaeSrcDirName, "dub"));
400         mkdirRecurse(buildPath(reggaeSrcDirName, "rules"));
401         mkdirRecurse(buildPath(reggaeSrcDirName, "backend"));
402         mkdirRecurse(buildPath(reggaeSrcDirName, "core", "rules"));
403     }
404 
405     //this foreach has to happen at compile time due
406     //to the string import below.
407     foreach(fileName; aliasSeqOf!(fileNames ~ foreignFiles)) {
408         auto file = File(reggaeSrcFileName(options, fileName), "w");
409         file.write(import(fileName));
410     }
411 
412     writeConfig(output, options);
413 }
414 
415 
416 private void writeConfig(T)(auto ref T output, in Options options) {
417 
418     import reggae.io: log;
419 
420     output.log("Writing reggae configuration");
421 
422     auto file = File(reggaeSrcFileName(options, "config.d"), "w");
423 
424     file.writeln(q{
425 module reggae.config;
426 import reggae.ctaa;
427 import reggae.types;
428 import reggae.options;
429     });
430 
431     version(minimal) file.writeln("enum isDubProject = false;");
432     file.writeln("immutable options = ", options, ";");
433 
434     file.writeln("enum userVars = AssocList!(string, string)([");
435     foreach(key, value; options.userVars) {
436         file.writeln("assocEntry(`", key, "`, `", value, "`), ");
437     }
438     file.writeln("]);");
439 
440     try {
441         writeDubConfig(output, options, file);
442     } catch(Exception ex) {
443         stderr.writeln("Could not write dub configuration, try 'dub upgrade': ", ex.msg);
444         throw ex;
445     }
446 }
447 
448 
449 private string reggaeSrcFileName(in Options options, in string fileName) @safe pure nothrow {
450     return buildPath(reggaeSrcDirName(options), fileName);
451 }