1 /**
2  * Copyright: Copyright Jason White, 2016
3  * License:   MIT
4  * Authors:   Jason White
5  */
6 module button.resource;
7 import std.digest : DigestType, isDigest;
8 
9 import std.array : Appender;
10 
11 /**
12  * A resource identifier.
13  */
14 alias ResourceId = string;
15 
16 /**
17  * A resource key must be unique.
18  */
19 struct ResourceKey
20 {
21     /**
22      * File path to the resource. To ensure uniqueness, this should never be
23      * changed after construction.
24      */
25     string path;
26 
27     /**
28      * Compares this key with another.
29      */
30     int opCmp()(const auto ref typeof(this) that) const pure nothrow
31     {
32         import std.algorithm.comparison : cmp;
33         return cmp(this.path, that.path);
34     }
35 }
36 
37 unittest
38 {
39     static assert(ResourceKey("abc") == ResourceKey("abc"));
40     static assert(ResourceKey("abc") < ResourceKey("abcd"));
41 }
42 
43 /**
44  * Compute the checksum of a file.
45  */
46 private DigestType!Hash digestFile(Hash)(string path)
47     if (isDigest!Hash)
48 {
49     import std.digest : digest;
50     import io.file : SysException, File, FileFlags;
51     import io.range : byChunk;
52 
53     ubyte[4096] buf;
54 
55     try
56     {
57         return digest!Hash(File(path, FileFlags.readExisting).byChunk(buf));
58     }
59     catch (SysException e)
60     {
61         // This may fail if the given path is a directory. The path could have
62         // also been deleted.
63         return typeof(return).init;
64     }
65 }
66 
67 /**
68  * Computes a stable checksum for the given directory.
69  *
70  * Note that we cannot use std.file.dirEntries here. dirEntries() yields the
71  * full path to the directory entries. We only want the file name, not the path
72  * to it. Thus, we're forced to list the directory contents the old fashioned
73  * way.
74  */
75 version (Posix)
76 private DigestType!Hash digestDir(Hash)(const(char)* path)
77     if (isDigest!Hash)
78 {
79     import core.stdc.string : strlen;
80     import std.array : Appender;
81     import std.algorithm.sorting : sort;
82     import core.sys.posix.dirent : DIR, dirent, opendir, closedir, readdir;
83 
84     Appender!(string[]) entries;
85 
86     if (DIR* dir = opendir(path))
87     {
88         scope (exit) closedir(dir);
89 
90         while (true)
91         {
92             dirent* entry = readdir(dir);
93             if (!entry) break;
94 
95             entries.put(entry.d_name[0 .. strlen(entry.d_name.ptr)].idup);
96         }
97     }
98     else
99     {
100         // In this case, this is either not a directory or it doesn't exist.
101         return typeof(return).init;
102     }
103 
104     // The order in which files are listed is not guaranteed to be sorted.
105     // Whether or not it is sorted depends on the file system implementation.
106     // Thus, we sort them to eliminate that potential source of non-determinism.
107     sort(entries.data);
108 
109     Hash digest;
110     digest.start();
111 
112     foreach (name; entries.data)
113     {
114         digest.put(cast(const(ubyte)[])name);
115         digest.put(cast(ubyte)0); // Null terminator
116     }
117 
118     return digest.finish();
119 }
120 
121 /**
122  * Computes a stable checksum for the given directory.
123  */
124 private DigestType!Hash digestDir(Hash)(string path)
125     if (isDigest!Hash)
126 {
127     import std.internal.cstring : tempCString;
128 
129     return digestDir!Hash(path.tempCString());
130 }
131 
132 /**
133  * A representation of a file on the disk.
134  */
135 struct Resource
136 {
137     import std.datetime : SysTime;
138     import std.digest.sha : SHA256;
139 
140     /**
141      * Digest to use to determine changes.
142      */
143     alias Hash = SHA256;
144 
145     enum Status
146     {
147         // The state of the resource is not known.
148         unknown,
149 
150         // The path does not exist on disk.
151         missing,
152 
153         // The path refers to a file.
154         file,
155 
156         // The path refers to a directory.
157         directory,
158     }
159 
160     /**
161      * File path to the resource. To ensure uniqueness, this should never be
162      * changed after construction.
163      */
164     ResourceId path;
165 
166     /**
167      * Status of the file.
168      */
169     Status status = Status.unknown;
170 
171     /**
172      * Checksum of the file.
173      */
174     DigestType!Hash checksum;
175 
176     this(ResourceId path, Status status = Status.unknown,
177             const(ubyte[]) checksum = []) pure
178     {
179         import std.algorithm.comparison : min;
180 
181         this.path = path;
182         this.status = status;
183 
184         // The only times the length will be different are:
185         //  - The database is corrupt
186         //  - The digest length changed
187         // In either case, it doesn't matter. If the checksum changes it will
188         // simply be recomputed and order will once again be restored in the
189         // realm.
190         immutable bytes = min(this.checksum.length, checksum.length);
191         this.checksum[0 .. bytes] = checksum[0 .. bytes];
192     }
193 
194     /**
195      * Returns a string representation of this resource. This is just the path
196      * to the resource.
197      */
198     string toString() const pure nothrow
199     {
200         return path;
201     }
202 
203     /**
204      * Returns a short string representation of the path.
205      */
206     @property string toShortString() const pure nothrow
207     {
208         import std.path : baseName;
209         return path.baseName;
210     }
211 
212     /**
213      * Returns the unique identifier for this vertex.
214      */
215     @property inout(ResourceId) identifier() inout pure nothrow
216     {
217         return path;
218     }
219 
220     /**
221      * Compares the file path of this resource with another.
222      */
223     int opCmp()(const auto ref Resource rhs) const pure
224     {
225         import std.path : filenameCmp;
226         return filenameCmp(this.path, rhs.path);
227     }
228 
229     /// Ditto
230     bool opEquals()(const auto ref Resource rhs) const pure
231     {
232         return opCmp(rhs) == 0;
233     }
234 
235     unittest
236     {
237         assert(Resource("a") < Resource("b"));
238         assert(Resource("b") > Resource("a"));
239 
240         assert(Resource("test", Resource.Status.unknown) ==
241                Resource("test", Resource.Status.unknown));
242         assert(Resource("test", Resource.Status.file) ==
243                Resource("test", Resource.Status.directory));
244     }
245 
246     /**
247      * Updates the last modified time and checksum of this resource. Returns
248      * true if anything changed.
249      *
250      * Note that the checksum is not recomputed if the modification time is the
251      * same.
252      */
253     bool update()
254     {
255         version (Posix)
256         {
257             import core.sys.posix.sys.stat : lstat, stat_t, S_IFMT, S_IFDIR,
258                    S_IFREG;
259             import io.file.stream : SysException;
260             import core.stdc.errno : errno, ENOENT;
261             import std.datetime : unixTimeToStdTime;
262             import std.internal.cstring : tempCString;
263 
264             stat_t statbuf = void;
265 
266             auto tmpPath = path.tempCString();
267 
268             Status newStatus;
269             DigestType!Hash newChecksum;
270 
271             if (lstat(tmpPath, &statbuf) != 0)
272             {
273                 if (errno == ENOENT)
274                     newStatus = Status.missing;
275                 else
276                     throw new SysException("Failed to stat resource");
277             }
278             else if ((statbuf.st_mode & S_IFMT) == S_IFREG)
279             {
280                 newChecksum = digestFile!Hash(path);
281                 newStatus = Status.file;
282             }
283             else if ((statbuf.st_mode & S_IFMT) == S_IFDIR)
284             {
285                 newChecksum = digestDir!Hash(tmpPath);
286                 newStatus = Status.directory;
287             }
288             else
289             {
290                 // The resource is neither a file nor a directory. It could be a
291                 // special file such as a FIFO, block device, etc. In those
292                 // cases, we cannot be expected to track changes to those types
293                 // of files.
294                 newStatus = Status.unknown;
295             }
296 
297             if (newStatus != status || checksum != newChecksum)
298             {
299                 status = newStatus;
300                 checksum = newChecksum;
301                 return true;
302             }
303 
304             return false;
305         }
306         else
307         {
308             static assert(false, "Not implemented yet.");
309         }
310     }
311 
312     /**
313      * Returns true if the status of this resource is known.
314      */
315     @property bool statusKnown() const pure nothrow
316     {
317         return status != Status.unknown;
318     }
319 
320     /**
321      * Deletes the resource from disk.
322      */
323     void remove(bool dryRun) nothrow
324     {
325         import std.file : unlink = remove, isFile;
326         import io;
327 
328         // Only delete this file if we know about it. This helps prevent the
329         // build system from haphazardly deleting files that were added to the
330         // build description but never output by a task.
331         if (!statusKnown)
332             return;
333 
334         // TODO: Use rmdir instead if this is a directory.
335 
336         if (!dryRun)
337         {
338             try
339             {
340                 unlink(path);
341             }
342             catch (Exception e)
343             {
344             }
345         }
346 
347         status = Status.missing;
348     }
349 }
350 
351 /**
352  * Normalizes a resource path while trying to make it relative to the buildRoot.
353  * If it cannot be done, the path is made absolute.
354  *
355  * Params:
356  *     buildRoot = The root directory of the build. Probably always the current
357  *                 working directory.
358  *     taskDir   = The working directory of the task this is for. The path is
359  *                 normalized relative to this directory.
360  *     path      = The path to be normalized.
361  */
362 string normPath(const(char)[] buildRoot, const(char)[] taskDir,
363         const(char)[] path) pure
364 {
365     import std.path : isAbsolute, buildNormalizedPath, pathSplitter,
366            filenameCmp, dirSeparator;
367     import std.algorithm.searching : skipOver;
368     import std.algorithm.iteration : joiner;
369     import std.array : array;
370     import std.utf : byChar;
371 
372     auto normalized = buildNormalizedPath(taskDir, path);
373 
374     // If the normalized path is absolute, get a relative path if the absolute
375     // path is inside the working directory. This is done instead of always
376     // getting a relative path because we don't want to get relative paths to
377     // directories like "/usr/include". If the build directory moves, absolute
378     // paths outside will become invalid.
379     if (isAbsolute(normalized) && buildRoot.length)
380     {
381         auto normPS  = pathSplitter(normalized);
382         auto buildPS = pathSplitter(buildRoot);
383 
384         alias pred = (a, b) => filenameCmp(a, b) == 0;
385 
386         if (skipOver!pred(normPS, &buildPS) && buildPS.empty)
387             return normPS.joiner(dirSeparator).byChar.array;
388     }
389 
390     return normalized;
391 }
392 
393 pure unittest
394 {
395     version (Posix)
396     {
397         assert(normPath("", "", "foo") == "foo");
398         assert(normPath("", "foo", "bar") == "foo/bar");
399 
400         assert(normPath("", "foo/../foo/.", "bar/../baz") == "foo/baz");
401 
402         assert(normPath("", "foo", "/usr/include/bar") == "/usr/include/bar");
403         assert(normPath("/usr", "foo", "/usr/bar") == "bar");
404         assert(normPath("/usr/include", "foo", "/usr/bar") == "/usr/bar");
405     }
406 }
407 
408 /**
409  * Output range of implicit resources.
410  *
411  * This is used to easily accumulate implicit resources while also normalizing
412  * their paths at the same time.
413  */
414 struct Resources
415 {
416     import std.array : Appender;
417     import std.range : isInputRange, ElementType;
418 
419     Appender!(Resource[]) resources;
420 
421     alias resources this;
422 
423     string buildDir;
424     string taskDir;
425 
426     this(string buildDir, string taskDir)
427     {
428         this.buildDir = buildDir;
429         this.taskDir = taskDir;
430     }
431 
432     void put(R)(R items)
433         if (isInputRange!R && is(ElementType!R : const(char)[]))
434     {
435         import std.range : empty, popFront, front;
436 
437         for (; !items.empty; items.popFront())
438             put(items.front);
439     }
440 
441     void put(const(char)[] item)
442     {
443         resources.put(Resource(normPath(buildDir, taskDir, item)));
444     }
445 
446     void put(R)(R items)
447         if (isInputRange!R && is(ElementType!R : Resource))
448     {
449         resources.put(items);
450     }
451 
452     void put(Resource item)
453     {
454         item.path = normPath(buildDir, taskDir, item.path);
455         resources.put(item);
456     }
457 }