1 /** 2 * Copyright: Copyright Jason White, 2016 3 * License: MIT 4 * Authors: Jason White 5 */ 6 module button.resource; 7 import std.digest : DigestType, isDigest; 8 9 import std.array : Appender; 10 11 /** 12 * A resource identifier. 13 */ 14 alias ResourceId = string; 15 16 /** 17 * A resource key must be unique. 18 */ 19 struct ResourceKey 20 { 21 /** 22 * File path to the resource. To ensure uniqueness, this should never be 23 * changed after construction. 24 */ 25 string path; 26 27 /** 28 * Compares this key with another. 29 */ 30 int opCmp()(const auto ref typeof(this) that) const pure nothrow 31 { 32 import std.algorithm.comparison : cmp; 33 return cmp(this.path, that.path); 34 } 35 } 36 37 unittest 38 { 39 static assert(ResourceKey("abc") == ResourceKey("abc")); 40 static assert(ResourceKey("abc") < ResourceKey("abcd")); 41 } 42 43 /** 44 * Compute the checksum of a file. 45 */ 46 private DigestType!Hash digestFile(Hash)(string path) 47 if (isDigest!Hash) 48 { 49 import std.digest : digest; 50 import io.file : SysException, File, FileFlags; 51 import io.range : byChunk; 52 53 ubyte[4096] buf; 54 55 try 56 { 57 return digest!Hash(File(path, FileFlags.readExisting).byChunk(buf)); 58 } 59 catch (SysException e) 60 { 61 // This may fail if the given path is a directory. The path could have 62 // also been deleted. 63 return typeof(return).init; 64 } 65 } 66 67 /** 68 * Computes a stable checksum for the given directory. 69 * 70 * Note that we cannot use std.file.dirEntries here. dirEntries() yields the 71 * full path to the directory entries. We only want the file name, not the path 72 * to it. Thus, we're forced to list the directory contents the old fashioned 73 * way. 74 */ 75 version (Posix) 76 private DigestType!Hash digestDir(Hash)(const(char)* path) 77 if (isDigest!Hash) 78 { 79 import core.stdc.string : strlen; 80 import std.array : Appender; 81 import std.algorithm.sorting : sort; 82 import core.sys.posix.dirent : DIR, dirent, opendir, closedir, readdir; 83 84 Appender!(string[]) entries; 85 86 if (DIR* dir = opendir(path)) 87 { 88 scope (exit) closedir(dir); 89 90 while (true) 91 { 92 dirent* entry = readdir(dir); 93 if (!entry) break; 94 95 entries.put(entry.d_name[0 .. strlen(entry.d_name.ptr)].idup); 96 } 97 } 98 else 99 { 100 // In this case, this is either not a directory or it doesn't exist. 101 return typeof(return).init; 102 } 103 104 // The order in which files are listed is not guaranteed to be sorted. 105 // Whether or not it is sorted depends on the file system implementation. 106 // Thus, we sort them to eliminate that potential source of non-determinism. 107 sort(entries.data); 108 109 Hash digest; 110 digest.start(); 111 112 foreach (name; entries.data) 113 { 114 digest.put(cast(const(ubyte)[])name); 115 digest.put(cast(ubyte)0); // Null terminator 116 } 117 118 return digest.finish(); 119 } 120 121 /** 122 * Computes a stable checksum for the given directory. 123 */ 124 private DigestType!Hash digestDir(Hash)(string path) 125 if (isDigest!Hash) 126 { 127 import std.internal.cstring : tempCString; 128 129 return digestDir!Hash(path.tempCString()); 130 } 131 132 /** 133 * A representation of a file on the disk. 134 */ 135 struct Resource 136 { 137 import std.datetime : SysTime; 138 import std.digest.sha : SHA256; 139 140 /** 141 * Digest to use to determine changes. 142 */ 143 alias Hash = SHA256; 144 145 enum Status 146 { 147 // The state of the resource is not known. 148 unknown, 149 150 // The path does not exist on disk. 151 missing, 152 153 // The path refers to a file. 154 file, 155 156 // The path refers to a directory. 157 directory, 158 } 159 160 /** 161 * File path to the resource. To ensure uniqueness, this should never be 162 * changed after construction. 163 */ 164 ResourceId path; 165 166 /** 167 * Status of the file. 168 */ 169 Status status = Status.unknown; 170 171 /** 172 * Checksum of the file. 173 */ 174 DigestType!Hash checksum; 175 176 this(ResourceId path, Status status = Status.unknown, 177 const(ubyte[]) checksum = []) pure 178 { 179 import std.algorithm.comparison : min; 180 181 this.path = path; 182 this.status = status; 183 184 // The only times the length will be different are: 185 // - The database is corrupt 186 // - The digest length changed 187 // In either case, it doesn't matter. If the checksum changes it will 188 // simply be recomputed and order will once again be restored in the 189 // realm. 190 immutable bytes = min(this.checksum.length, checksum.length); 191 this.checksum[0 .. bytes] = checksum[0 .. bytes]; 192 } 193 194 /** 195 * Returns a string representation of this resource. This is just the path 196 * to the resource. 197 */ 198 string toString() const pure nothrow 199 { 200 return path; 201 } 202 203 /** 204 * Returns a short string representation of the path. 205 */ 206 @property string toShortString() const pure nothrow 207 { 208 import std.path : baseName; 209 return path.baseName; 210 } 211 212 /** 213 * Returns the unique identifier for this vertex. 214 */ 215 @property inout(ResourceId) identifier() inout pure nothrow 216 { 217 return path; 218 } 219 220 /** 221 * Compares the file path of this resource with another. 222 */ 223 int opCmp()(const auto ref Resource rhs) const pure 224 { 225 import std.path : filenameCmp; 226 return filenameCmp(this.path, rhs.path); 227 } 228 229 /// Ditto 230 bool opEquals()(const auto ref Resource rhs) const pure 231 { 232 return opCmp(rhs) == 0; 233 } 234 235 unittest 236 { 237 assert(Resource("a") < Resource("b")); 238 assert(Resource("b") > Resource("a")); 239 240 assert(Resource("test", Resource.Status.unknown) == 241 Resource("test", Resource.Status.unknown)); 242 assert(Resource("test", Resource.Status.file) == 243 Resource("test", Resource.Status.directory)); 244 } 245 246 /** 247 * Updates the last modified time and checksum of this resource. Returns 248 * true if anything changed. 249 * 250 * Note that the checksum is not recomputed if the modification time is the 251 * same. 252 */ 253 bool update() 254 { 255 version (Posix) 256 { 257 import core.sys.posix.sys.stat : lstat, stat_t, S_IFMT, S_IFDIR, 258 S_IFREG; 259 import io.file.stream : SysException; 260 import core.stdc.errno : errno, ENOENT; 261 import std.datetime : unixTimeToStdTime; 262 import std.internal.cstring : tempCString; 263 264 stat_t statbuf = void; 265 266 auto tmpPath = path.tempCString(); 267 268 Status newStatus; 269 DigestType!Hash newChecksum; 270 271 if (lstat(tmpPath, &statbuf) != 0) 272 { 273 if (errno == ENOENT) 274 newStatus = Status.missing; 275 else 276 throw new SysException("Failed to stat resource"); 277 } 278 else if ((statbuf.st_mode & S_IFMT) == S_IFREG) 279 { 280 newChecksum = digestFile!Hash(path); 281 newStatus = Status.file; 282 } 283 else if ((statbuf.st_mode & S_IFMT) == S_IFDIR) 284 { 285 newChecksum = digestDir!Hash(tmpPath); 286 newStatus = Status.directory; 287 } 288 else 289 { 290 // The resource is neither a file nor a directory. It could be a 291 // special file such as a FIFO, block device, etc. In those 292 // cases, we cannot be expected to track changes to those types 293 // of files. 294 newStatus = Status.unknown; 295 } 296 297 if (newStatus != status || checksum != newChecksum) 298 { 299 status = newStatus; 300 checksum = newChecksum; 301 return true; 302 } 303 304 return false; 305 } 306 else 307 { 308 static assert(false, "Not implemented yet."); 309 } 310 } 311 312 /** 313 * Returns true if the status of this resource is known. 314 */ 315 @property bool statusKnown() const pure nothrow 316 { 317 return status != Status.unknown; 318 } 319 320 /** 321 * Deletes the resource from disk. 322 */ 323 void remove(bool dryRun) nothrow 324 { 325 import std.file : unlink = remove, isFile; 326 import io; 327 328 // Only delete this file if we know about it. This helps prevent the 329 // build system from haphazardly deleting files that were added to the 330 // build description but never output by a task. 331 if (!statusKnown) 332 return; 333 334 // TODO: Use rmdir instead if this is a directory. 335 336 if (!dryRun) 337 { 338 try 339 { 340 unlink(path); 341 } 342 catch (Exception e) 343 { 344 } 345 } 346 347 status = Status.missing; 348 } 349 } 350 351 /** 352 * Normalizes a resource path while trying to make it relative to the buildRoot. 353 * If it cannot be done, the path is made absolute. 354 * 355 * Params: 356 * buildRoot = The root directory of the build. Probably always the current 357 * working directory. 358 * taskDir = The working directory of the task this is for. The path is 359 * normalized relative to this directory. 360 * path = The path to be normalized. 361 */ 362 string normPath(const(char)[] buildRoot, const(char)[] taskDir, 363 const(char)[] path) pure 364 { 365 import std.path : isAbsolute, buildNormalizedPath, pathSplitter, 366 filenameCmp, dirSeparator; 367 import std.algorithm.searching : skipOver; 368 import std.algorithm.iteration : joiner; 369 import std.array : array; 370 import std.utf : byChar; 371 372 auto normalized = buildNormalizedPath(taskDir, path); 373 374 // If the normalized path is absolute, get a relative path if the absolute 375 // path is inside the working directory. This is done instead of always 376 // getting a relative path because we don't want to get relative paths to 377 // directories like "/usr/include". If the build directory moves, absolute 378 // paths outside will become invalid. 379 if (isAbsolute(normalized) && buildRoot.length) 380 { 381 auto normPS = pathSplitter(normalized); 382 auto buildPS = pathSplitter(buildRoot); 383 384 alias pred = (a, b) => filenameCmp(a, b) == 0; 385 386 if (skipOver!pred(normPS, &buildPS) && buildPS.empty) 387 return normPS.joiner(dirSeparator).byChar.array; 388 } 389 390 return normalized; 391 } 392 393 pure unittest 394 { 395 version (Posix) 396 { 397 assert(normPath("", "", "foo") == "foo"); 398 assert(normPath("", "foo", "bar") == "foo/bar"); 399 400 assert(normPath("", "foo/../foo/.", "bar/../baz") == "foo/baz"); 401 402 assert(normPath("", "foo", "/usr/include/bar") == "/usr/include/bar"); 403 assert(normPath("/usr", "foo", "/usr/bar") == "bar"); 404 assert(normPath("/usr/include", "foo", "/usr/bar") == "/usr/bar"); 405 } 406 } 407 408 /** 409 * Output range of implicit resources. 410 * 411 * This is used to easily accumulate implicit resources while also normalizing 412 * their paths at the same time. 413 */ 414 struct Resources 415 { 416 import std.array : Appender; 417 import std.range : isInputRange, ElementType; 418 419 Appender!(Resource[]) resources; 420 421 alias resources this; 422 423 string buildDir; 424 string taskDir; 425 426 this(string buildDir, string taskDir) 427 { 428 this.buildDir = buildDir; 429 this.taskDir = taskDir; 430 } 431 432 void put(R)(R items) 433 if (isInputRange!R && is(ElementType!R : const(char)[])) 434 { 435 import std.range : empty, popFront, front; 436 437 for (; !items.empty; items.popFront()) 438 put(items.front); 439 } 440 441 void put(const(char)[] item) 442 { 443 resources.put(Resource(normPath(buildDir, taskDir, item))); 444 } 445 446 void put(R)(R items) 447 if (isInputRange!R && is(ElementType!R : Resource)) 448 { 449 resources.put(items); 450 } 451 452 void put(Resource item) 453 { 454 item.path = normPath(buildDir, taskDir, item.path); 455 resources.put(item); 456 } 457 }