1 /**
2  The main entry point for the reggae tool. Its tasks are:
3  $(UL
4    $(LI Verify that a $(D reggafile.d) exists in the selected directory)
5    $(LI Generate a $(D reggaefile.d) for dub projects)
6    $(LI Write out the reggae library files and $(D config.d))
7    $(LI Compile the build description with the reggae library files to produce $(D buildgen))
8    $(LI Produce $(D dcompile), a binary to call the D compiler to obtain dependencies during compilation)
9    $(LI Call the produced $(D buildgen) binary)
10  )
11  */
12 
13 module reggae.reggae;
14 
15 import std.stdio;
16 import std.process: execute, environment;
17 import std.array: array, join, empty, split;
18 import std.path: absolutePath, buildPath, relativePath;
19 import std.typetuple;
20 import std.file;
21 import std.conv: text;
22 import std.exception: enforce;
23 import std.conv: to;
24 import std.algorithm;
25 
26 import reggae.options;
27 import reggae.ctaa;
28 import reggae.types;
29 import reggae.file;
30 
31 
32 version(minimal) {
33     //empty stubs for minimal version of reggae
34     void maybeCreateReggaefile(T...)(T) {}
35     void writeDubConfig(T...)(T) {}
36 } else {
37     import reggae.dub.interop;
38 }
39 
40 mixin template reggaeGen(targets...) {
41     mixin buildImpl!targets;
42     mixin ReggaeMain;
43 }
44 
45 mixin template ReggaeMain() {
46     import reggae.options: getOptions;
47     import std.stdio: stdout, stderr;
48 
49     int main(string[] args) {
50         try {
51             run(stdout, args);
52         } catch(Exception ex) {
53             stderr.writeln(ex.msg);
54             return 1;
55         }
56 
57         return 0;
58     }
59 }
60 
61 void run(T)(auto ref T output, string[] args) {
62     auto options = getOptions(args);
63     run(output, options);
64 }
65 
66 void run(T)(auto ref T output, Options options) {
67     if(options.earlyExit) return;
68     enforce(options.projectPath != "", "A project path must be specified");
69 
70     // write out the library source files to be compiled/interpreted
71     // with the user's build description
72     writeSrcFiles(output, options);
73 
74     if(options.isJsonBuild) {
75         immutable haveToReturn = jsonBuild(options);
76         if(haveToReturn) return;
77     }
78 
79     maybeCreateReggaefile(output, options);
80     createBuild(output, options);
81 }
82 
83 //get JSON description of the build from a scripting language
84 //and transform it into a build description
85 //return true if no D files are present
86 bool jsonBuild(Options options) {
87     immutable jsonOutput = getJsonOutput(options);
88     return jsonBuild(options, jsonOutput);
89 }
90 
91 //transform JSON description into a Build struct
92 //return true if no D files are present
93 bool jsonBuild(Options options, in string jsonOutput) {
94     enforce(options.backend != Backend.binary, "Binary backend not supported via JSON");
95 
96     version(minimal)
97         assert(0, "JSON builds not supported in minimal version");
98     else {
99         import reggae.json_build;
100         import reggae.buildgen;
101         import reggae.rules.common: Language;
102 
103         auto build = jsonToBuild(options.projectPath, jsonOutput);
104         doBuild(build, jsonToOptions(options, jsonOutput));
105 
106         import reggae.buildgen:writeCompilationDB;
107         if(!options.noCompilationDB) writeCompilationDB(build, options);
108 
109         //true -> exit early
110         return !build.targets.canFind!(a => a.getLanguage == Language.D);
111     }
112 }
113 
114 
115 private string getJsonOutput(in Options options) @safe {
116     const args = getJsonOutputArgs(options);
117     const path = environment.get("PATH", "").split(":");
118     const pythonPaths = environment.get("PYTHONPATH", "").split(":");
119     const nodePaths = environment.get("NODE_PATH", "").split(":");
120     const luaPaths = environment.get("LUA_PATH", "").split(";");
121     const srcDir = buildPath(options.workingDir, hiddenDir, "src");
122     const binDir = buildPath(srcDir, "reggae");
123     auto env = ["PATH": (path ~ binDir).join(":"),
124                 "PYTHONPATH": (pythonPaths ~ srcDir).join(":"),
125                 "NODE_PATH": (nodePaths ~ options.projectPath ~ binDir).join(":"),
126                 "LUA_PATH": (luaPaths ~ buildPath(options.projectPath, "?.lua") ~ buildPath(binDir, "?.lua")).join(";")];
127     immutable res = execute(args, env);
128     enforce(res.status == 0, text("Could not execute ", args.join(" "), ":\n", res.output));
129     return res.output;
130 }
131 
132 private string[] getJsonOutputArgs(in Options options) @safe {
133 
134     import std.process: environment;
135     import std.json: parseJSON;
136 
137     final switch(options.reggaeFileLanguage) {
138 
139     case BuildLanguage.D:
140         assert(0, "Cannot obtain JSON build for builds written in D");
141 
142     case BuildLanguage.Python:
143 
144         auto optionsString = () @trusted {
145             import std.json;
146             import std.traits;
147             auto jsonVal = parseJSON(`{}`);
148             foreach(member; __traits(allMembers, typeof(options))) {
149                 static if(is(typeof(mixin(`options.` ~ member)) == const(Backend)) ||
150                           is(typeof(mixin(`options.` ~ member)) == const(string)) ||
151                           is(typeof(mixin(`options.` ~ member)) == const(bool)) ||
152                           is(typeof(mixin(`options.` ~ member)) == const(string[string])) ||
153                           is(typeof(mixin(`options.` ~ member)) == const(string[])))
154                     jsonVal.object[member] = mixin(`options.` ~ member);
155             }
156             return jsonVal.toString;
157         }();
158 
159         const haveReggaePython = "REGGAE_PYTHON" in environment;
160         auto pythonParts = haveReggaePython
161             ? [environment["REGGAE_PYTHON"]]
162             : ["/usr/bin/env", "python"];
163         return pythonParts ~ ["-B", "-m", "reggae.reggae_json_build",
164                 "--options", optionsString,
165                 options.projectPath];
166 
167     case BuildLanguage.Ruby:
168         return ["ruby", "-S",
169                 "-I" ~ options.projectPath,
170                 "-I" ~ buildPath(options.workingDir, hiddenDir, "src", "reggae"),
171                 "reggae_json_build.rb"];
172 
173     case BuildLanguage.Lua:
174         return ["lua", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.lua")];
175 
176     case BuildLanguage.JavaScript:
177         return ["node", buildPath(options.workingDir, hiddenDir, "src", "reggae", "reggae_json_build.js")];
178     }
179 }
180 
181 enum coreFiles = [
182     "options.d",
183     "buildgen_main.d", "buildgen.d",
184     "build.d",
185     "backend/package.d", "backend/binary.d",
186     "package.d", "range.d", "reflect.d",
187     "dependencies.d", "types.d", "dcompile.d",
188     "ctaa.d", "sorting.d", "file.d",
189     "rules/package.d",
190     "rules/common.d",
191     "rules/d.d",
192     "rules/c_and_cpp.d",
193     "core/package.d", "core/rules/package.d",
194     ];
195 enum otherFiles = [
196     "backend/ninja.d", "backend/make.d", "backend/tup.d",
197     "dub/info.d", "rules/dub.d",
198     ];
199 
200 version(minimal) {
201     enum string[] foreignFiles = [];
202 } else {
203     enum foreignFiles = [
204         "__init__.py", "build.py", "reflect.py", "rules.py", "reggae_json_build.py",
205         "reggae.rb", "reggae_json_build.rb",
206         "reggae-js.js", "reggae_json_build.js",
207         "JSON.lua", "reggae.lua", "reggae_json_build.lua",
208         ];
209 }
210 
211 //all files that need to be written out and compiled
212 private string[] fileNames() @safe pure nothrow {
213     version(minimal)
214         return coreFiles;
215     else
216         return coreFiles ~ otherFiles;
217 }
218 
219 
220 private void createBuild(T)(auto ref T output, in Options options) {
221 
222     import reggae.io: log;
223 
224     enforce(options.reggaeFilePath.exists, text("Could not find ", options.reggaeFilePath));
225 
226     //compile the binaries (the build generator and dcompile)
227     immutable buildGenName = compileBinaries(output, options);
228 
229     //binary backend has no build generator, it _is_ the build
230     if(options.backend == Backend.binary) return;
231 
232     //only got here to build .dcompile
233     if(options.isScriptBuild) return;
234 
235     //actually run the build generator
236     output.log("Running the created binary to generate the build");
237     immutable retRunBuildgen = execute([buildPath(options.workingDir, hiddenDir, buildGenName)]);
238     enforce(retRunBuildgen.status == 0,
239             text("Couldn't execute the produced ", buildGenName, " binary:\n", retRunBuildgen.output));
240 
241     if(retRunBuildgen.output.length )output.log(retRunBuildgen.output);
242 }
243 
244 
245 struct Binary {
246     string name;
247     const(string)[] cmd;
248 }
249 
250 
251 private string compileBinaries(T)(auto ref T output, in Options options) {
252     buildDCompile(output, options);
253 
254     immutable buildGenName = getBuildGenName(options);
255     if(options.isScriptBuild) return buildGenName;
256 
257     const buildGenCmd = getCompileBuildGenCmd(options);
258     immutable buildObjName = "build.o";
259     buildBinary(output, options, Binary(buildObjName, buildGenCmd));
260 
261     const reggaeFileDeps = getReggaeFileDependenciesDlang;
262     auto objFiles = [buildObjName];
263     if(!reggaeFileDeps.empty) {
264         immutable rest = "rest.o";
265         buildBinary(output,
266                     options,
267                     Binary(rest,
268                            [options.dCompiler,
269                             "-c",
270                             "-of" ~ "rest.o"] ~
271                            importPaths(options) ~
272                            reggaeFileDeps));
273         objFiles ~= rest;
274     }
275 
276     buildBinary(output,
277                 options,
278                 Binary(buildGenName,
279                        [options.dCompiler, "-of" ~ buildGenName] ~ objFiles));
280 
281     return buildGenName;
282 }
283 
284 void buildDCompile(T)(auto ref T output, in Options options) {
285     if(!thisExePath.newerThan(buildPath(options.workingDir, hiddenDir, "dcompile")))
286         return;
287 
288     immutable cmd = [options.dCompiler,
289                      "-Isrc",
290                      "-ofdcompile",
291                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dcompile.d"),
292                      buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName, "dependencies.d")];
293 
294     buildBinary(output, options, Binary("dcompile", cmd));
295 }
296 
297 private bool isExecutable(in char[] path) @trusted nothrow @nogc //TODO: @safe
298 {
299     import core.sys.posix.unistd;
300     import std.internal.cstring;
301     return (access(path.tempCString(), X_OK) == 0);
302 }
303 
304 private void buildBinary(T)(auto ref T output, in Options options, in Binary bin) {
305     import reggae.io: log;
306     import std.process;
307 
308     string[string] env;
309     auto config = Config.none;
310     auto maxOutput = size_t.max;
311     auto workDir = buildPath(options.workingDir, hiddenDir);
312     const extraInfo = options.verbose ? " with " ~  bin.cmd.join(" ") : "";
313     output.log("Compiling metabuild binary ", bin.name, extraInfo);
314     // std.process.execute has a bug where using workDir and a relative path
315     // don't work (https://issues.dlang.org/show_bug.cgi?id=15915)
316     // so executeShell is used instead
317     immutable res = executeShell(bin.cmd.join(" "), env, config, maxOutput, workDir);
318     enforce(res.status == 0, text("Couldn't execute ", bin.cmd.join(" "), "\nin ", workDir,
319                                   ":\n", res.output,
320                                   "\n", "bin.name: ", bin.name, ", bin.cmd: ", bin.cmd.join(" ")));
321 
322 }
323 
324 
325 private const(string)[] getCompileBuildGenCmd(in Options options) @safe {
326     import reggae.rules.common: objExt;
327 
328     const reggaeSrcs = ("config.d" ~ fileNames).
329         filter!(a => a != "dcompile.d").
330         map!(a => buildPath(reggaeSrcRelDirName, a)).array;
331 
332     immutable buildBinFlags = options.backend == Backend.binary
333         ? ["-O", "-inline"]
334         : [];
335     const commonBefore = ["./dcompile",
336                           "--objFile=" ~ "build.o",
337                           "--depFile=" ~ "reggaefile.dep",
338                           options.dCompiler] ~
339         importPaths(options) ~
340         ["-g",
341          "-debug"];
342     const commonAfter = buildBinFlags ~
343         options.reggaeFilePath ~ reggaeSrcs;
344     version(minimal) return commonBefore ~ "-version=minimal" ~ commonAfter;
345     else return commonBefore ~ commonAfter;
346 }
347 
348 private string[] importPaths(in Options options) @safe nothrow {
349     import std.file;
350 
351     immutable srcDir = "-I" ~ buildPath("src");
352     // if compiling phobos, the includes for the reggaefile.d compilation
353     // will pick up the new phobos if we include the src path
354     return "std".exists ? [srcDir] : ["-I" ~ options.projectPath, srcDir];
355 }
356 
357 private string getBuildGenName(in Options options) @safe pure nothrow {
358     return options.backend == Backend.binary ? buildPath("..", "build") : "buildgen";
359 }
360 
361 
362 immutable reggaeSrcRelDirName = buildPath("src", "reggae");
363 
364 string reggaeSrcDirName(in Options options) @safe pure nothrow {
365     return buildPath(options.workingDir, hiddenDir, reggaeSrcRelDirName);
366 }
367 
368 
369 void writeSrcFiles(T)(auto ref T output, in Options options) {
370     import reggae.io: log;
371 
372     output.log("Writing reggae source files");
373 
374     import std.file: mkdirRecurse;
375     immutable reggaeSrcDirName = reggaeSrcDirName(options);
376     if(!reggaeSrcDirName.exists) {
377         mkdirRecurse(reggaeSrcDirName);
378         mkdirRecurse(buildPath(reggaeSrcDirName, "dub"));
379         mkdirRecurse(buildPath(reggaeSrcDirName, "rules"));
380         mkdirRecurse(buildPath(reggaeSrcDirName, "backend"));
381         mkdirRecurse(buildPath(reggaeSrcDirName, "core", "rules"));
382     }
383 
384     //this foreach has to happen at compile time due
385     //to the string import below.
386     foreach(fileName; aliasSeqOf!(fileNames ~ foreignFiles)) {
387         auto file = File(reggaeSrcFileName(options, fileName), "w");
388         file.write(import(fileName));
389     }
390 
391     writeConfig(output, options);
392 }
393 
394 
395 private void writeConfig(T)(auto ref T output, in Options options) {
396 
397     import reggae.io: log;
398 
399     output.log("Writing reggae configuration");
400 
401     auto file = File(reggaeSrcFileName(options, "config.d"), "w");
402 
403     file.writeln(q{
404 module reggae.config;
405 import reggae.ctaa;
406 import reggae.types;
407 import reggae.options;
408     });
409 
410     version(minimal) file.writeln("enum isDubProject = false;");
411     file.writeln("immutable options = ", options, ";");
412 
413     file.writeln("enum userVars = AssocList!(string, string)([");
414     foreach(key, value; options.userVars) {
415         file.writeln("assocEntry(`", key, "`, `", value, "`), ");
416     }
417     file.writeln("]);");
418 
419     try {
420         writeDubConfig(output, options, file);
421     } catch(Exception ex) {
422         stderr.writeln("Could not write dub configuration, try 'dub upgrade': ", ex.msg);
423         throw ex;
424     }
425 }
426 
427 
428 private string reggaeSrcFileName(in Options options, in string fileName) @safe pure nothrow {
429     return buildPath(reggaeSrcDirName(options), fileName);
430 }