1 /++ 2 Standalone Mongo driver implementation. 3 4 Copyright: 2020 Symmetry Investments with portion (c) 2012-2016 Nicolas Gurrola 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 +/ 7 module kaleidic.mongo_standalone; 8 9 import std.array; 10 import std.bitmanip; 11 import std.socket; 12 13 static immutable string mongoDriverName = "mongo-standalone"; 14 static immutable string mongoDriverVersion = "0.0.4"; 15 16 // just a demo of how it can be used 17 version(none) 18 void main() { 19 20 /+ 21 string omg; 22 foreach(a; 0 .. 8000) 23 omg ~= "a"; 24 25 26 document doc1 = document([ 27 bson_value("foo", 12), 28 bson_value("bar", 12.4), 29 bson_value("baz", 12L), 30 bson_value("faz", omg), 31 bson_value("eaz", RegEx("asdasdsa", "i")), 32 bson_value("asds", null), 33 bson_value("sub", document([ 34 bson_value("sub1", 12) 35 ])), 36 ]); 37 +/ 38 39 //auto connection = new MongoConnection("mongodb://testuser:testpassword@localhost/test?slaveOk=true"); 40 auto connection = new MongoConnection("mongodb://localhost/test?slaveOk=true", "testuser", "testpassword"); 41 42 //connection.insert(false, "test.world", [doc1]); 43 44 import std.stdio; 45 //writeln(connection.query(0, "world", 0, 1, document.init)); 46 auto answer = connection.query("test.world", 35, 1, document.init, document.init, 1); 47 writeln(answer); 48 } 49 50 enum QueryFlags { 51 TailableCursor = (1 << 1), 52 SlaveOk = (1 << 2), 53 NoCursorTimeout = (1 << 4), 54 AwaitData = (1 << 5), 55 Exhaust = (1 << 6), 56 Partial = (1 << 7) 57 } 58 59 /// Server Wire version indicating supported features 60 /// $(LINK https://github.com/mongodb/specifications/blob/master/source/wireversion-featurelist.rst) 61 enum WireVersion { 62 /// Pre 2.6 (-2.4) 63 old = 0, 64 /// Server version 2.6 65 v26 = 1, 66 /// Server version 2.6 67 v26_2 = 2, 68 /// Server version 3.0 69 v30 = 3, 70 /// Server version 3.2 71 v32 = 4, 72 /// Server version 3.4 73 v34 = 5, 74 /// Server version 3.6 75 v36 = 6, 76 /// Server version 4.0 77 v40 = 7, 78 /// Server version 4.2 79 v42 = 8, 80 } 81 82 class MongoConnection { 83 84 Socket socket; 85 IReceiveStream stream; 86 87 uint defaultQueryFlags; 88 89 /// Reported minimum wire version by the server 90 WireVersion minWireVersion; 91 /// Reported maximum wire version by the server 92 WireVersion maxWireVersion; 93 94 private document handshake(string authDatabase, string username, 95 document application) { 96 import std.compiler : compilerName = name, version_major, version_minor; 97 import std.conv : text, to; 98 import std.system : os; 99 100 auto dbcmd = authDatabase ~ ".$cmd"; 101 102 static immutable osType = os.to!string; 103 104 version (X86_64) 105 static immutable osArchitecture = "x86_64"; 106 else version (X86) 107 static immutable osArchitecture = "x86"; 108 else version (ARM) 109 static immutable osArchitecture = "arm"; 110 else version (PPC64) 111 static immutable osArchitecture = "ppc64"; 112 else version (PPC) 113 static immutable osArchitecture = "ppc"; 114 else 115 static assert(false, "no name for this architecture"); 116 117 static immutable platform = text(compilerName, " v", version_major, ".", version_minor); 118 119 bson_value[] client; 120 if (application.length) 121 { 122 auto name = application["name"]; 123 if (name == bson_value.init) 124 throw new Exception("Given application without name"); 125 // https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#limitations 126 if (name.toString().length > 128) 127 throw new Exception("Application name must not exceed 128 bytes"); 128 129 client ~= bson_value("application", application); 130 } 131 132 client ~= [ 133 bson_value("driver", document([ 134 bson_value("name", mongoDriverName), 135 bson_value("version", mongoDriverVersion) 136 ])), 137 bson_value("os", document([ 138 bson_value("type", osType), 139 bson_value("architecture", osArchitecture) 140 ])), 141 bson_value("platform", platform) 142 ]; 143 144 auto cmd = [ 145 bson_value("isMaster", 1), 146 bson_value("client", document(client)) 147 ]; 148 if (username.length) 149 cmd ~= bson_value("saslSupportedMechs", authDatabase ~ "." ~ username); 150 151 auto reply = query(dbcmd, 0, -1, document(cmd)); 152 if (reply.documents.length != 1 || reply.documents[0]["ok"].get!double != 1) 153 throw new Exception("MongoDB Handshake failed"); 154 155 return reply.documents[0]; 156 } 157 158 private void authenticateScramSha1(string authDatabase, string username, 159 string password) { 160 auto dbcmd = authDatabase ~ ".$cmd"; 161 162 bson_value conversationId; 163 164 ScramState state; 165 166 ubyte[] payload = cast(ubyte[]) state.createInitialRequest(username); 167 168 auto cmd = document([ 169 bson_value("saslStart", 1), 170 bson_value("mechanism", "SCRAM-SHA-1"), 171 bson_value("payload", payload), 172 bson_value("options", document([ 173 bson_value("skipEmptyExchange", true) 174 ])) 175 ]); 176 177 178 auto firstReply = query(dbcmd, 0, -1, cmd); 179 if(firstReply.documents.length != 1 || firstReply.documents[0]["ok"].get!double != 1) 180 throw new Exception("Auth failed at first step (username)"); 181 182 conversationId = cast(bson_value) firstReply.documents[0]["conversationId"]; 183 184 auto response = firstReply.documents[0]["payload"].get!(const(ubyte[])); 185 186 const digest = makeDigest(username, password); 187 188 payload = cast(typeof(payload)) state.update(digest, cast(string) response); 189 190 cmd = document([ 191 bson_value("saslContinue", 1), 192 bson_value("conversationId", conversationId), 193 bson_value("payload", payload) 194 ]); 195 196 auto secondReply = query(dbcmd, 0, -1, cmd); 197 if(secondReply.documents.length != 1) 198 throw new Exception("Auth error at second step"); 199 200 if(secondReply.documents[0]["ok"].get!double != 1) 201 throw new Exception("Auth failed at second step (password)"); 202 203 auto response2 = secondReply.documents[0]["payload"].get!(const(ubyte[])); 204 205 payload = cast(typeof(payload)) state.finalize(cast(string) response2); 206 207 // newer servers can save a roundtrip (they know the password here already) 208 if(secondReply.documents[0]["done"].get!bool) 209 return; 210 211 cmd = document([ 212 bson_value("saslContinue", 1), 213 bson_value("conversationId", conversationId), 214 bson_value("payload", payload) 215 ]); 216 217 auto finalReply = query(dbcmd, 0, -1, cmd); 218 if(finalReply.documents.length != 1) 219 throw new Exception("Auth error at final step"); 220 221 if(finalReply.documents[0]["ok"].get!double != 1) 222 throw new Exception("Auth failed at final step"); 223 224 if(!finalReply.documents[0]["done"].get!bool) 225 throw new Exception("Authentication didn't respond 'done'"); 226 } 227 228 /++ 229 Connects to a Mongo database. You can give a username and password in the 230 connection string if URL encoded, OR you can pass it as arguments to this 231 constructor. 232 233 Please note that username/password in the connectionString will OVERRIDE 234 ones given as separate arguments. 235 +/ 236 this(string connectionString, string defaultUsername = null, string defaultPassword = null) { 237 import std.uri : decodeComponent; 238 import std.string; 239 240 auto uri = Uri(connectionString); 241 242 if(uri.hosts.length == 0) 243 throw new Exception("no host given in connection string"); 244 245 foreach(ref p; uri.hosts) { 246 if(p.port == 0) 247 p.port = 27017; 248 } 249 250 string host = decodeComponent(uri.hosts[0].host); 251 if(host.length == 0) 252 host = "localhost"; 253 254 // FIXME: break up the host into the list and at least 255 // pretend to care about replication sets 256 257 string authDb = "admin"; 258 string appName; 259 260 string username = defaultUsername; 261 string password = defaultPassword; 262 263 auto split = uri.userinfo.indexOf(":"); 264 if(split != -1) { 265 username = decodeComponent(uri.userinfo[0 .. split]); 266 password = decodeComponent(uri.userinfo[split + 1 .. $]); 267 } 268 if(uri.path.length > 1) 269 authDb = uri.path[1 .. $]; 270 271 foreach(part; uri.query.splitter("&")) { 272 split = part.indexOf("="); 273 auto name = decodeComponent(part[0 .. split]); 274 auto value = decodeComponent(part[split + 1 .. $]); 275 276 // https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options 277 switch(name) { 278 case "slaveOk": defaultQueryFlags |= QueryFlags.SlaveOk; break; 279 case "appName": appName = value; break; 280 default: throw new Exception("Unsupported mongo db connect option: " ~ name); 281 } 282 } 283 284 if(host[0] == '/') { 285 version(Posix) { 286 socket = new Socket(AddressFamily.UNIX, SocketType.STREAM); 287 socket.connect(new UnixAddress(host)); 288 } else throw new Exception("Cannot use unix socket on Windows at this time"); 289 } else { 290 socket = new TcpSocket(new InternetAddress(host, cast(ushort) uri.hosts[0].port)); 291 } 292 293 stream = new SocketReceiveStream(socket); 294 295 document appArgs; 296 if (appName.length) 297 appArgs = document([bson_value("name", appName)]); 298 auto handshakeResponse = handshake(authDb, username, appArgs); 299 300 minWireVersion = cast(WireVersion)handshakeResponse["minWireVersion"].get!int; 301 maxWireVersion = cast(WireVersion)handshakeResponse["maxWireVersion"].get!int; 302 303 bool supportsScramSha1 = maxWireVersion >= WireVersion.v30; 304 bool supportsScramSha256 = maxWireVersion >= WireVersion.v40; 305 306 auto saslSupportedMechs = handshakeResponse["saslSupportedMechs"]; 307 if (saslSupportedMechs != bson_value.init) { 308 auto arr = saslSupportedMechs.get!(const(bson_value)[]); 309 supportsScramSha1 = false; 310 supportsScramSha256 = false; 311 foreach (v; arr) { 312 switch (v.toString) { 313 case "SCRAM-SHA-1": 314 supportsScramSha1 = true; 315 break; 316 case "SCRAM-SHA-256": 317 supportsScramSha256 = true; 318 break; 319 default: 320 // unsupported mechanism 321 break; 322 } 323 } 324 } 325 326 // https://github.com/mongodb/specifications/blob/master/source/auth/auth.rst#supported-authentication-methods 327 if (username.length) { 328 // TODO: support other (certificate based) authentication mechanisms 329 330 // TODO: SCRAM-SHA-256 support 331 // if (supportsScramSha256) { 332 // } else 333 if (supportsScramSha1) { 334 authenticateScramSha1(authDb, username, password); 335 } else { 336 if (maxWireVersion < WireVersion.v30) 337 throw new Exception("legacy MONGODB-CR authentication not implemented"); 338 else 339 throw new Exception( 340 "Cannot authenticate because no common authentication mechanism could be found."); 341 } 342 } 343 } 344 345 void update(const(char)[] fullCollectionName, bool upsert, bool multiupdate, document selector, document update) { 346 MsgHeader header; 347 348 header.requestID = ++nextRequestId; 349 header.opCode = OP.UPDATE; 350 351 SendBuffer sb; 352 353 sb.add(header); 354 355 int zero; 356 sb.add(zero); 357 358 sb.add(fullCollectionName); 359 int flags; 360 if(upsert) flags |= 1; 361 if(multiupdate) flags |= 2; 362 sb.add(flags); 363 sb.add(selector); 364 sb.add(update); 365 366 send(sb.data); 367 } 368 369 void insert(bool continueOnError, const(char)[] fullCollectionName, document[] documents) { 370 MsgHeader header; 371 372 header.requestID = ++nextRequestId; 373 header.opCode = OP.INSERT; 374 375 SendBuffer sb; 376 377 sb.add(header); 378 379 int flags = continueOnError ? 1 : 0; 380 sb.add(flags); 381 sb.add(fullCollectionName); 382 foreach(doc; documents) 383 sb.add(doc); 384 385 send(sb.data); 386 } 387 388 OP_REPLY getMore(const(char)[] fullCollectionName, int numberToReturn, long cursorId, int limitTotalEntries = int.max) { 389 MsgHeader header; 390 391 header.requestID = ++nextRequestId; 392 header.opCode = OP.GET_MORE; 393 394 SendBuffer sb; 395 396 sb.add(header); 397 398 int zero; 399 sb.add(zero); 400 sb.add(fullCollectionName); 401 sb.add(numberToReturn); 402 sb.add(cursorId); 403 404 send(sb.data); 405 406 auto reply = stream.readReply(); 407 408 reply.numberToReturn = numberToReturn; 409 reply.fullCollectionName = fullCollectionName; 410 reply.current = 0; 411 reply.connection = this; 412 413 return reply; 414 } 415 416 void delete_(const(char)[] fullCollectionName, bool singleRemove, document selector) { 417 MsgHeader header; 418 419 header.requestID = ++nextRequestId; 420 header.opCode = OP.DELETE; 421 422 SendBuffer sb; 423 424 sb.add(header); 425 426 int zero; 427 sb.add(0); 428 sb.add(fullCollectionName); 429 int flags = singleRemove ? 1 : 0; 430 sb.add(flags); 431 sb.add(selector); 432 433 send(sb.data); 434 } 435 436 void killCursors(const(long)[] cursorIds) { 437 MsgHeader header; 438 439 header.requestID = ++nextRequestId; 440 header.opCode = OP.KILL_CURSORS; 441 442 SendBuffer sb; 443 444 sb.add(header); 445 int zero = 0; 446 sb.add(zero); 447 448 sb.add(cast(int) cursorIds.length); 449 foreach(id; cursorIds) 450 sb.add(id); 451 452 send(sb.data); 453 } 454 455 OP_REPLY query(const(char)[] fullCollectionName, int numberToSkip, int numberToReturn, document query, document returnFieldsSelector = document.init, int flags = 1, int limitTotalEntries = int.max) { 456 if(flags == 1) 457 flags = defaultQueryFlags; 458 MsgHeader header; 459 460 header.requestID = ++nextRequestId; 461 header.opCode = OP.QUERY; 462 463 SendBuffer sb; 464 465 sb.add(header); 466 467 sb.add(flags); 468 sb.add(fullCollectionName); 469 sb.add(numberToSkip); 470 sb.add(numberToReturn); 471 sb.add(query); 472 473 if(returnFieldsSelector != document.init) 474 sb.add(returnFieldsSelector); 475 476 send(sb.data); 477 478 auto reply = stream.readReply(); 479 480 reply.numberToReturn = numberToReturn; 481 reply.limitRemaining = limitTotalEntries; 482 reply.fullCollectionName = fullCollectionName; 483 reply.current = 0; 484 reply.connection = this; 485 486 return reply; 487 } 488 489 private int nextRequestId; 490 491 private void send(const(ubyte)[] data) { 492 while(data.length) { 493 auto ret = socket.send(data); 494 if(ret <= 0) 495 throw new Exception("wtf"); 496 data = data[ret .. $]; 497 } 498 } 499 } 500 501 unittest { 502 auto uri = Uri("mongo://test:12,foo:14/"); 503 assert(uri.hosts.length == 2); 504 assert(uri.hosts[0] == UriHost("test", 12)); 505 assert(uri.hosts[1] == UriHost("foo", 14)); 506 } 507 508 interface IReceiveStream { 509 final OP_REPLY readReply() { 510 OP_REPLY reply; 511 512 513 reply.header.messageLength = readInt(); 514 reply.header.requestID = readInt(); 515 reply.header.responseTo = readInt(); 516 reply.header.opCode = readInt(); 517 518 // flag 0: cursor not found. 1: query failure. $err set in a thing. 519 reply.responseFlags = readInt(); 520 reply.cursorID = readLong(); 521 reply.startingFrom = readInt(); 522 reply.numberReturned = readInt(); 523 524 reply.documents.length = reply.numberReturned; 525 526 foreach(ref doc; reply.documents) { 527 doc = readBson(); 528 } 529 530 return reply; 531 } 532 533 final document readBson() { 534 document d; 535 d.bytesCount = readInt(); 536 537 int remaining = d.bytesCount; 538 remaining -= 4; // length 539 remaining -= 1; // terminating zero 540 541 while(remaining > 0) { 542 d.values_ ~= readBsonValue(remaining); 543 } 544 545 d.terminatingZero = readByte(); 546 if(d.terminatingZero != 0) 547 throw new Exception("something wrong reading bson"); 548 549 return d; 550 } 551 552 final bson_value readBsonValue(ref int remaining) { 553 bson_value v; 554 555 v.tag = readByte(); 556 remaining--; 557 558 v.e_name = readZeroTerminatedChars(); 559 remaining -= v.e_name.length + 1; // include the zero term 560 561 switch(v.tag) { 562 case 0x00: // not supposed to exist! 563 throw new Exception("invalid bson"); 564 case 0x01: 565 auto d = readDouble(); 566 remaining -= 8; 567 v.x01 = d; 568 break; 569 case 0x02: 570 v.x02 = readCharsWithLengthPrefix(); 571 remaining -= 5 + v.x02.length; // length and zero 572 break; 573 case 0x03: 574 v.x03 = readBson(); 575 remaining -= v.x03.bytesCount; 576 break; 577 case 0x04: 578 v.x04 = readBson(); 579 remaining -= v.x04.bytesCount; 580 break; 581 case 0x05: 582 auto length = readInt(); 583 v.x05_tag = readByte(); 584 v.x05_data = readBytes(length); 585 remaining -= v.x05_data.length + 5; 586 break; 587 case 0x06: // undefined 588 // intentionally blank, no additional data 589 break; 590 case 0x07: 591 foreach(ref b; v.x07) { 592 b = readByte(); 593 remaining--; 594 } 595 break; 596 case 0x08: 597 v.x08 = readByte() ? true : false; 598 remaining--; 599 break; 600 case 0x09: 601 v.x09 = readLong(); 602 remaining -= 8; 603 break; 604 case 0x0a: // null 605 // intentionally blank, no additional data 606 break; 607 case 0x0b: 608 v.x0b_regex = readZeroTerminatedChars(); 609 remaining -= v.x0b_regex.length + 1; 610 v.x0b_flags = readZeroTerminatedChars(); 611 remaining -= v.x0b_flags.length + 1; 612 break; 613 case 0x0d: 614 v.x0d = readCharsWithLengthPrefix(); 615 remaining -= 5 + v.x0d.length; // length and zero 616 break; 617 case 0x10: 618 v.x10 = readInt(); 619 remaining -= 4; 620 break; 621 case 0x11: 622 v.x11 = readLong(); 623 remaining -= 8; 624 break; 625 case 0x12: 626 v.x12 = readLong(); 627 remaining -= 8; 628 break; 629 case 0x13: 630 foreach(ref b; v.x13) { 631 b = readByte(); 632 remaining--; 633 } 634 break; 635 default: 636 import std.conv; 637 assert(0, "unsupported tag in bson: " ~ to!string(v.tag)); 638 } 639 640 return v; 641 } 642 643 final ubyte readByte() { 644 return readSmall!1[0]; 645 } 646 647 final int readInt() { 648 return readSmall!4.littleEndianToNative!int; 649 } 650 651 final long readLong() { 652 return readSmall!8.littleEndianToNative!long; 653 } 654 655 final double readDouble() { 656 return readSmall!8.littleEndianToNative!double; 657 } 658 659 final string readZeroTerminatedChars() { 660 return cast(string) readUntilNull(); 661 } 662 663 final string readCharsWithLengthPrefix() { 664 auto length = readInt(); 665 // length prefixed string allows 0 characters 666 auto bytes = readBytes(length); 667 if (bytes[$ - 1] != 0) 668 throw new Exception("Malformed length prefixed string"); 669 return cast(string)bytes[0 .. $ - 1]; 670 } 671 672 /// Reads exactly the amount of bytes specified and returns a newly 673 /// allocated buffer containing the data. 674 final immutable(ubyte)[] readBytes(size_t length) { 675 ubyte[] ret = new ubyte[length]; 676 readExact(ret); 677 return (() @trusted => cast(immutable)ret)(); 678 } 679 680 /// Reads a small number of bytes into a stack allocated array. Convenience 681 /// function calling $(LREF readExact). 682 ubyte[n] readSmall(size_t n)() { 683 ubyte[n] buf; 684 readExact(buf[]); 685 return buf; 686 } 687 688 /// Reads exactly the expected length into the given buffer argument. 689 void readExact(scope ubyte[] buffer); 690 691 /// Reads until a 0 byte and returns all data before it as newly allocated 692 /// buffer containing the data. 693 immutable(ubyte)[] readUntilNull(); 694 } 695 696 class SocketReceiveStream : IReceiveStream { 697 this(Socket socket) { 698 this.socket = socket; 699 } 700 701 Socket socket; 702 ubyte[4096] buffer; 703 // if bufferLength > bufferPos then following data is linear after bufferPos 704 // if bufferLength < bufferPos then data is following bufferPos and also from 0..bufferLength 705 // if bufferLength == bufferPos then all data is read 706 int bufferLength; 707 // may only increment until at the end of buffer.length, where it wraps back to 0 708 int bufferPos; 709 710 protected ptrdiff_t receiveSocket(ubyte[] buffer) { 711 const length = socket.receive(buffer); 712 713 if (length == 0) 714 throw new Exception("Remote side has closed the connection"); 715 else if (length == Socket.ERROR) 716 throw new Exception(lastSocketError()); 717 else 718 return length; 719 } 720 721 /// Loads more data after bufferPos if there is space, otherwise load before 722 /// bufferPos, indicating data is wrapping around. 723 void loadMore() { 724 if (bufferPos == bufferLength) { 725 // we read up all the bytes, start back from 0 for bigger recv buffer 726 bufferPos = 0; 727 bufferLength = 0; 728 } 729 730 ptrdiff_t length; 731 if (bufferLength < bufferPos) { 732 // length wrapped around before pos 733 // try to read up to the byte before bufferPos to not trigger the 734 // bufferPos == bufferLength case 735 length = receiveSocket(buffer[bufferLength .. bufferPos - 1]); 736 } else { 737 length = receiveSocket(buffer[bufferLength .. $ - 1]); 738 } 739 740 bufferLength += length; 741 if (bufferLength == buffer.length) 742 bufferLength = 0; 743 } 744 745 void readExact(scope ubyte[] dst) { 746 if (!dst.length) 747 return; 748 749 if (bufferPos == bufferLength) 750 loadMore(); 751 752 auto end = bufferPos + dst.length; 753 if (end < buffer.length) { 754 // easy linear copy 755 // receive until bufferLength wrapped around or we have enough bytes 756 while (bufferLength >= bufferPos && end > bufferLength) 757 loadMore(); 758 dst[] = buffer[bufferPos .. bufferPos = cast(typeof(bufferPos))end]; 759 } else { 760 // copy from wrapping buffer 761 size_t i; 762 while (i < dst.length) { 763 auto remaining = dst.length - i; 764 if (bufferPos < bufferLength) { 765 auto d = min(remaining, bufferLength - bufferPos); 766 dst[i .. i += d] = buffer[bufferPos .. bufferPos += d]; 767 } else if (bufferPos > bufferLength) { 768 auto d = buffer.length - bufferPos; 769 if (remaining >= d) { 770 dst[i .. i += d] = buffer[bufferPos .. $]; 771 dst[i .. i += bufferLength] = buffer[0 .. bufferPos = bufferLength]; 772 } else { 773 dst[i .. i += remaining] = buffer[bufferPos .. bufferPos += remaining]; 774 } 775 } else { 776 loadMore(); 777 } 778 } 779 } 780 } 781 782 immutable(ubyte)[] readUntilNull() { 783 auto ret = appender!(immutable(ubyte)[]); 784 while (true) { 785 ubyte[] chunk; 786 if (bufferPos < bufferLength) { 787 chunk = buffer[bufferPos .. bufferLength]; 788 } else if (bufferPos > bufferLength) { 789 chunk = buffer[bufferPos .. $]; 790 } else { 791 loadMore(); 792 continue; 793 } 794 795 auto zero = chunk.countUntil(0); 796 if (zero == -1) { 797 ret ~= chunk; 798 bufferPos += chunk.length; 799 if (bufferPos == buffer.length) 800 bufferPos = 0; 801 } else { 802 ret ~= chunk[0 .. zero]; 803 bufferPos += zero + 1; 804 return ret.data; 805 } 806 } 807 } 808 } 809 810 unittest { 811 class MockedSocketReceiveStream : SocketReceiveStream { 812 int maxChunk = 64; 813 ubyte generator = 0; 814 815 this() { 816 super(null); 817 } 818 819 protected override ptrdiff_t receiveSocket(ubyte[] buffer) { 820 if (buffer.length >= maxChunk) { 821 buffer[0 .. maxChunk] = ++generator; 822 return maxChunk; 823 } else { 824 buffer[] = ++generator; 825 return buffer.length; 826 } 827 } 828 } 829 830 auto stream = new MockedSocketReceiveStream(); 831 stream.buffer[] = 0; 832 auto b = stream.readBytes(3); 833 assert(b == [1, 1, 1]); 834 assert(stream.buffer[0 .. 64].all!(a => a == 1)); 835 assert(stream.buffer[64 .. $].all!(a => a == 0)); 836 b = stream.readBytes(3); 837 assert(b == [1, 1, 1]); 838 assert(stream.buffer[64 .. $].all!(a => a == 0)); 839 assert(stream.bufferPos == 6); 840 assert(stream.bufferLength == 64); 841 b = stream.readBytes(64); 842 assert(b[0 .. $ - 6].all!(a => a == 1)); 843 assert(b[$ - 6 .. $].all!(a => a == 2)); 844 assert(stream.bufferPos == 70); 845 assert(stream.bufferLength == 128); 846 b = stream.readBytes(57); 847 assert(b.all!(a => a == 2)); 848 assert(stream.bufferPos == 127); 849 assert(stream.bufferLength == 128); 850 b = stream.readBytes(8000); 851 assert(b[0] == 2); 852 assert(b[1 .. 65].all!(a => a == 3)); 853 assert(b[65 .. 129].all!(a => a == 4)); 854 855 stream.buffer[] = 0; 856 stream.bufferPos = stream.bufferLength = 0; 857 stream.generator = 0; 858 assert(stream.readUntilNull().length == 64 * 255); 859 } 860 861 struct SendBuffer { 862 private ubyte[4096] backing; 863 864 private ubyte[] buffer; 865 private size_t position; 866 867 ubyte[] data() { 868 assert(position > 4); 869 870 resetSize(); 871 return buffer[0 .. position]; 872 } 873 874 private void size(size_t addition) { 875 if(buffer is null) 876 buffer = backing[]; 877 if(position + addition > buffer.length) 878 buffer.length = buffer.length * 2; 879 } 880 881 void add(const document d) { 882 SendBuffer sb; 883 884 sb.add(d.bytesCount); 885 foreach(v; d.values) 886 sb.add(v); 887 sb.addByte(0); 888 889 auto data = sb.data; 890 //import std.stdio; writefln("%(%x %)",data); 891 892 this.add(data); 893 } 894 895 void add(const MsgHeader header) { 896 add(header.messageLength); 897 add(header.requestID); 898 add(header.responseTo); 899 add(header.opCode); 900 } 901 902 void add(const bson_value v) { 903 addByte(v.tag); 904 add(v.name); 905 906 static struct visitor { 907 this(SendBuffer* buf) { this.buf = buf; } 908 SendBuffer* buf; 909 void visit(long v) { buf.add(v); } 910 void visit(int v) { buf.add(v); } 911 void visit(bool v) { buf.addByte(v ? 1 : 0); } 912 void visit(double v) { buf.add(v); } 913 void visit(const document v) { buf.add(v); } 914 void visit(const(char)[] v) { buf.add(cast(int) v.length + 1); buf.add(v); } 915 void visit(const typeof(null)) { } 916 void visit(ubyte tag, const(ubyte)[] v) { buf.add(cast(int) v.length); buf.addByte(tag); buf.add(v); } 917 void visit(const Undefined v) { } 918 void visit(const ObjectId v) { buf.add(v.v[]); } 919 void visit(const Javascript v) { buf.add(cast(int) v.v.length + 1); buf.add(v.v); } 920 void visit(const Timestamp v) { buf.add(v.v); } 921 void visit(const UtcTimestamp v) { buf.add(v.v); } 922 void visit(const Decimal128 v) { buf.add(v.v[]); } 923 void visit(const RegEx v) { buf.add(v.regex); buf.add(v.flags); } 924 925 void visit(const(bson_value)[] v) { buf.add(document(v)); } 926 } 927 928 visitor mv = visitor(&this); 929 v.visit(mv); 930 } 931 932 933 void addByte(ubyte a) { 934 size(1); 935 buffer[position++] = (a >> 0) & 0xff; 936 } 937 938 void add(double i) { 939 long a = *(cast(long*) &i); 940 add(a); 941 } 942 943 void add(int a) { 944 size(4); 945 buffer[position++] = (a >> 0) & 0xff; 946 buffer[position++] = (a >> 8) & 0xff; 947 buffer[position++] = (a >> 16) & 0xff; 948 buffer[position++] = (a >> 24) & 0xff; 949 } 950 951 void add(long a) { 952 size(8); 953 buffer[position++] = (a >> 0) & 0xff; 954 buffer[position++] = (a >> 8) & 0xff; 955 buffer[position++] = (a >> 16) & 0xff; 956 buffer[position++] = (a >> 24) & 0xff; 957 buffer[position++] = (a >> 32) & 0xff; 958 buffer[position++] = (a >> 40) & 0xff; 959 buffer[position++] = (a >> 48) & 0xff; 960 buffer[position++] = (a >> 56) & 0xff; 961 } 962 963 // does NOT write the length out first! 964 void add(const(char)[] a) { 965 size(a.length + 1); 966 buffer[position .. position + a.length] = cast(ubyte[]) a[]; 967 position += a.length; 968 buffer[position++] = 0; 969 } 970 971 // does NOT write the length out first! 972 void add(const(ubyte)[] a) { 973 size(a.length); 974 buffer[position .. position + a.length] = cast(ubyte[]) a[]; 975 position += a.length; 976 } 977 978 private void resetSize() { 979 auto sz = cast(int) position; 980 buffer[0] = (sz >> 0) & 0xff; 981 buffer[1] = (sz >> 8) & 0xff; 982 buffer[2] = (sz >> 16) & 0xff; 983 buffer[3] = (sz >> 24) & 0xff; 984 } 985 } 986 987 struct MsgHeader { 988 int messageLength; 989 int requestID; 990 int responseTo; 991 int opCode; 992 } 993 994 enum OP { 995 REPLY = 1, 996 UPDATE = 2001, 997 INSERT = 2002, 998 QUERY = 2004, // sends a reply 999 GET_MORE = 2005, // sends a reply 1000 DELETE = 2006, 1001 KILL_CURSORS = 2007, 1002 MSG = 2013 1003 } 1004 1005 struct OP_UPDATE { 1006 MsgHeader header; 1007 int zero; 1008 const(char)[] fullCollectionName; 1009 int flags; // bit 0 == upsert, bit 1 == multiupdate if 1010 document selector; 1011 document update; 1012 } 1013 1014 struct OP_INSERT { 1015 MsgHeader header; 1016 int flags; // bit 0 == ContinueOnError 1017 const(char)[] fullCollectionName; 1018 document[] documents; 1019 } 1020 1021 struct OP_QUERY { 1022 MsgHeader header; 1023 int flags; // SEE: https://docs.mongodb.com/manual/reference/mongodb-wire-protocol/#op-query 1024 const(char)[] fullCollectionName; 1025 int numberToSkip; 1026 int numberToReturn; 1027 document query; 1028 document returnFieldsSelector; // optional..... 1029 } 1030 1031 struct OP_GET_MORE { 1032 MsgHeader header; 1033 int zero; 1034 const(char)[] fullCollectionName; 1035 int numberToReturn; 1036 long cursorID; 1037 } 1038 1039 struct OP_DELETE { 1040 MsgHeader header; 1041 int zero; 1042 const(char)[] fullCollectionName; 1043 int flags; // bit 0 is single remove 1044 document selector; 1045 } 1046 1047 // If a cursor is read until exhausted (read until OP_QUERY or OP_GET_MORE returns zero for the cursor id), there is no need to kill the cursor. 1048 struct OP_KILL_CURSORS { 1049 MsgHeader header; 1050 int zero; 1051 int numberOfCursorIds; 1052 const(long)[] cursorIDs; 1053 } 1054 1055 /+ 1056 // in mongo 3.6 1057 struct OP_MSG { 1058 MsgHeader header, 1059 uint flagBits; 1060 Section[] 1061 } 1062 +/ 1063 1064 struct OP_REPLY { 1065 MsgHeader header; 1066 int responseFlags; // flag 0: cursor not found. 1: query failure. $err set in a thing. 1067 long cursorID; 1068 int startingFrom; 1069 int numberReturned; 1070 document[] documents; 1071 1072 /* range elements */ 1073 int numberToReturn; 1074 const(char)[] fullCollectionName; 1075 size_t current; 1076 MongoConnection connection; 1077 int limitRemaining = int.max; 1078 1079 string errorMessage; 1080 int errorCode; 1081 1082 @property bool empty() { 1083 return errorCode != 0 || numberReturned == 0 || limitRemaining <= 0; 1084 } 1085 1086 void popFront() { 1087 limitRemaining--; 1088 current++; 1089 if(current == numberReturned) { 1090 this = connection.getMore(fullCollectionName, numberToReturn, cursorID, limitRemaining); 1091 if(numberReturned == 1) { 1092 auto err = documents[0]["$err"]; 1093 auto ecode = documents[0]["code"]; 1094 if(err !is bson_value.init) { 1095 errorMessage = err.get!string; 1096 errorCode = ecode.get!int; 1097 } 1098 } 1099 } 1100 } 1101 1102 @property document front() { 1103 return documents[current]; 1104 } 1105 } 1106 1107 /* bson */ 1108 1109 struct document { 1110 private int bytesCount; 1111 private const(bson_value)[] values_; 1112 private ubyte terminatingZero; 1113 1114 const(bson_value) opIndex(string name) { 1115 foreach(value; values) 1116 if(value.name == name) 1117 return value; 1118 return bson_value.init; 1119 } 1120 1121 this(const(bson_value)[] values) { 1122 values_ = values; 1123 terminatingZero = 0; 1124 foreach(v; values) 1125 bytesCount += v.size; 1126 } 1127 1128 size_t length() const { 1129 return values_.length; 1130 } 1131 1132 const(bson_value)[] values() const { 1133 return values_; 1134 } 1135 1136 string toString(int indent = 0) const { 1137 string s; 1138 1139 s ~= "{\n"; 1140 1141 foreach(value; values) { 1142 foreach(i; 0 .. indent + 1) s ~= "\t"; 1143 s ~= value.name; 1144 s ~= ": "; 1145 s ~= value.toString(); 1146 s ~= "\n"; 1147 } 1148 1149 foreach(i; 0 .. indent) s ~= "\t"; 1150 s ~= "}\n"; 1151 1152 return s; 1153 } 1154 } 1155 1156 struct ObjectId { 1157 ubyte[12] v; 1158 string toString() const { 1159 import std.format; 1160 return format("ObjectId(%(%02x%))", v[]); 1161 } 1162 1163 this(const ubyte[12] v) { 1164 this.v = v; 1165 } 1166 1167 this(string s) { 1168 import std.algorithm; 1169 if(s.startsWith("ObjectId(")) 1170 s = s["ObjectId(".length .. $-1]; 1171 if(s.length != 24) 1172 throw new Exception("invalid object id: " ~ s); 1173 static int hexToDec(char c) { 1174 if(c >= '0' && c <= '9') 1175 return c - '0'; 1176 if(c >= 'A' && c <= 'F') 1177 return c - 'A' + 10; 1178 if(c >= 'a' && c <= 'f') 1179 return c - 'a' + 10; 1180 throw new Exception("Invalid hex char " ~ c); 1181 } 1182 foreach(ref b; v) { 1183 b = cast(ubyte) ((hexToDec(s[0]) << 4) | hexToDec(s[1])); 1184 s = s[2 .. $]; 1185 } 1186 } 1187 } 1188 struct UtcTimestamp { 1189 long v; 1190 } 1191 struct RegEx { 1192 const(char)[] regex; 1193 const(char)[] flags; 1194 } 1195 struct Javascript { 1196 string v; 1197 } 1198 struct Timestamp { 1199 ulong v; 1200 } 1201 struct Decimal128 { 1202 ubyte[16] v; 1203 } 1204 struct Undefined {} 1205 1206 1207 struct ToStringVisitor(T) if (isSomeString!T) { 1208 import std.conv; 1209 1210 // of course this could have been a template but I wrote it out long-form for copy/paste purposes 1211 T visit(const double v) { return to!T(v); } 1212 T visit(const string v) { return to!T(v); } 1213 T visit(const document v) { return to!T(v); } 1214 T visit(const const(bson_value)[] v) { return to!T(v); } 1215 T visit(const ubyte tag, const ubyte[] v) { return to!T(v); } 1216 T visit(const ObjectId v) { return to!T(v); } 1217 T visit(const bool v) { return to!T(v); } 1218 T visit(const UtcTimestamp v) { return to!T(v); } 1219 T visit(const typeof(null)) { return to!T(null); } 1220 T visit(const RegEx v) { return to!T(v); } 1221 T visit(const Javascript v) { return to!T(v); } 1222 T visit(const int v) { return to!T(v); } 1223 T visit(const Undefined) { return "undefined".to!T; } 1224 T visit(const Timestamp v) { return to!T(v); } 1225 T visit(const long v) { return to!T(v); } 1226 T visit(const Decimal128 v) { return to!T(v); } 1227 } 1228 1229 struct GetVisitor(T) { 1230 T visit(V)(const V t) { 1231 static if (isIntegral!V) { 1232 static if(isIntegral!T || isFloatingPoint!T) 1233 return cast(T)t; 1234 else throw new Exception("incompatible type"); 1235 } else static if (isFloatingPoint!V) { 1236 static if(isFloatingPoint!T) 1237 return cast(T)t; 1238 else throw new Exception("incompatible type"); 1239 } else { 1240 static if(is(V : T)) 1241 return t; 1242 else throw new Exception("incompatible type"); 1243 } 1244 } 1245 1246 T visit(ubyte tag, const(ubyte)[] v) { 1247 static if(is(typeof(v) : T)) 1248 return v; 1249 else throw new Exception("incompatible type " ~ T.stringof); 1250 } 1251 } 1252 1253 struct bson_value { 1254 private ubyte tag; 1255 private const(char)[] e_name; 1256 1257 // It only allows integer types or const getting in order to work right in the visitor... 1258 /// Tries to get the value matching exactly this type. The type will convert 1259 /// between different floating point types and integral types as well as 1260 /// perform a conversion from integral types to floating point types. 1261 T get(T)() const if (__traits(compiles, GetVisitor!T)) { 1262 GetVisitor!T v; 1263 return visit(v); 1264 } 1265 1266 const(char)[] name() const { 1267 return e_name; 1268 } 1269 1270 this(const(char)[] name, bson_value v) { 1271 this = v; 1272 e_name = name; 1273 } 1274 1275 this(const(char)[] name, double v) { 1276 e_name = name; 1277 tag = 0x01; 1278 x01 = v; 1279 } 1280 this(const(char)[] name, string v) { 1281 e_name = name; 1282 tag = 0x02; 1283 x02 = v; 1284 } 1285 this(const(char)[] name, document v) { 1286 e_name = name; 1287 tag = 0x03; 1288 x03 = v; 1289 } 1290 this(const(char)[] name, bson_value[] v) { 1291 e_name = name; 1292 tag = 0x04; 1293 const(bson_value)[] n; 1294 n.reserve(v.length); 1295 import std.conv; 1296 foreach(idx, i; v) 1297 n ~= bson_value(to!string(idx), i); 1298 x04 = document(n); 1299 } 1300 this(const(char)[] name, const(ubyte)[] v, ubyte tag = 0x00) { 1301 e_name = name; 1302 this.tag = 0x05; 1303 x05_tag = tag; 1304 x05_data = v; 1305 } 1306 this(const(char)[] name, ObjectId v) { 1307 e_name = name; 1308 tag = 0x07; 1309 x07 = v.v; 1310 } 1311 this(const(char)[] name, bool v) { 1312 e_name = name; 1313 tag = 0x08; 1314 x08 = v; 1315 } 1316 this(const(char)[] name, UtcTimestamp v) { 1317 e_name = name; 1318 tag = 0x09; 1319 x09 = v.v; 1320 } 1321 this(const(char)[] name, typeof(null)) { 1322 e_name = name; 1323 tag = 0x0a; 1324 } 1325 this(const(char)[] name, RegEx v) { 1326 e_name = name; 1327 tag = 0x0b; 1328 x0b_regex = v.regex; 1329 x0b_flags = v.flags; 1330 } 1331 this(const(char)[] name, Javascript v) { 1332 e_name = name; 1333 tag = 0x0d; 1334 x0d = v.v; 1335 } 1336 this(const(char)[] name, int v) { 1337 e_name = name; 1338 tag = 0x10; 1339 x10 = v; 1340 } 1341 this(const(char)[] name, Timestamp v) { 1342 e_name = name; 1343 tag = 0x11; 1344 x11 = v.v; 1345 } 1346 this(const(char)[] name, long v) { 1347 e_name = name; 1348 tag = 0x12; 1349 x12 = v; 1350 } 1351 this(const(char)[] name, Decimal128 v) { 1352 e_name = name; 1353 tag = 0x13; 1354 x13 = v.v; 1355 } 1356 1357 auto visit(T)(T t) const { 1358 switch(tag) { 1359 case 0x00: throw new Exception("invalid bson"); 1360 case 0x01: return t.visit(x01); 1361 case 0x02: return t.visit(x02); 1362 case 0x03: return t.visit(x03); 1363 case 0x04: return t.visit(x04.values); 1364 case 0x05: return t.visit(x05_tag, x05_data); 1365 case 0x06: return t.visit(Undefined()); 1366 case 0x07: return t.visit(ObjectId(x07)); 1367 case 0x08: return t.visit(x08); 1368 case 0x09: return t.visit(UtcTimestamp(x09)); 1369 case 0x0a: return t.visit(null); 1370 case 0x0b: return t.visit(RegEx(x0b_regex, x0b_flags)); 1371 case 0x0d: return t.visit(Javascript(x0d)); 1372 case 0x10: return t.visit(x10); 1373 case 0x11: return t.visit(Timestamp(x11)); 1374 case 0x12: return t.visit(x12); 1375 case 0x13: return t.visit(Decimal128(x13)); 1376 default: 1377 import std.conv; 1378 assert(0, "unsupported tag in bson: " ~ to!string(tag)); 1379 1380 } 1381 } 1382 1383 string toString() const { 1384 ToStringVisitor!string v; 1385 return visit(v); 1386 } 1387 1388 private union { 1389 void* zero; 1390 double x01; 1391 string x02; // given with an int length preceding and 0 terminator 1392 document x03; // child object 1393 document x04; // array with indexes as key values 1394 struct { 1395 ubyte x05_tag; 1396 const(ubyte)[] x05_data; // binary data 1397 } 1398 void* undefined; 1399 ubyte[12] x07; // a guid/ ObjectId 1400 bool x08; 1401 long x09; // utc datetime stamp 1402 void* x0a; // null 1403 struct { 1404 const(char)[] x0b_regex; 1405 const(char)[] x0b_flags; // alphabetized 1406 } 1407 string x0d; // javascript 1408 int x10; // integer!!! omg 1409 ulong x11; // timestamp 1410 long x12; // integer!!! 1411 ubyte[16] x13; // decimal 128 1412 } 1413 1414 size_t size() const { 1415 auto count = 2 + e_name.length; 1416 1417 switch(this.tag) { 1418 case 0x00: // not supposed to exist! 1419 throw new Exception("invalid bson"); 1420 case 0x01: 1421 count += 8; 1422 break; 1423 case 0x02: 1424 count += 5 + x02.length; // length and zero 1425 break; 1426 case 0x03: 1427 count += x03.bytesCount; 1428 break; 1429 case 0x04: 1430 count += x04.bytesCount; 1431 break; 1432 case 0x05: 1433 count += x05_data.length; 1434 count += 5; // length and tag 1435 break; 1436 case 0x06: // undefined 1437 // intentionally blank, no additional data 1438 break; 1439 case 0x07: 1440 count += x07.length; 1441 break; 1442 case 0x08: 1443 count++; 1444 break; 1445 case 0x09: 1446 count += 8; 1447 break; 1448 case 0x0a: // null 1449 // intentionally blank, no additional data 1450 break; 1451 case 0x0b: 1452 count += x0b_regex.length + 1; 1453 count += x0b_flags.length + 1; 1454 break; 1455 case 0x0d: 1456 count += 5 + x0d.length; // length and zero 1457 break; 1458 case 0x10: 1459 count += 4; 1460 break; 1461 case 0x11: 1462 count += 8; 1463 break; 1464 case 0x12: 1465 count += 8; 1466 break; 1467 case 0x13: 1468 count += x13.length; 1469 break; 1470 default: 1471 import std.conv; 1472 assert(0, "unsupported tag in bson: " ~ to!string(tag)); 1473 } 1474 1475 return count; 1476 } 1477 } 1478 1479 struct UriHost { 1480 string host; 1481 int port; 1482 } 1483 1484 /* copy/pasted from arsd.cgi, modified for mongo-specific comma behavior. used with permission */ 1485 struct Uri { 1486 // scheme//userinfo@host:port/path?query#fragment 1487 1488 string scheme; /// e.g. "http" in "http://example.com/" 1489 string userinfo; /// the username (and possibly a password) in the uri 1490 UriHost[] hosts; 1491 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 1492 string query; /// the stuff after the ? in a uri 1493 string fragment; /// the stuff after the # in a uri. 1494 1495 /// Breaks down a uri string to its components 1496 this(string uri) { 1497 reparse(uri); 1498 } 1499 1500 private void reparse(string uri) { 1501 // from RFC 3986 1502 // the ctRegex triples the compile time and makes ugly errors for no real benefit 1503 // it was a nice experiment but just not worth it. 1504 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 1505 /* 1506 Captures: 1507 0 = whole url 1508 1 = scheme, with : 1509 2 = scheme, no : 1510 3 = authority, with // 1511 4 = authority, no // 1512 5 = path 1513 6 = query string, with ? 1514 7 = query string, no ? 1515 8 = anchor, with # 1516 9 = anchor, no # 1517 */ 1518 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 1519 // instead, I will DIY and cut that down to 0.6s on the same computer. 1520 /* 1521 1522 Note that authority is 1523 user:password@domain:port 1524 where the user:password@ part is optional, and the :port is optional. 1525 1526 Regex translation: 1527 1528 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 1529 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 1530 Path cannot have any ? or # in it. It is optional. 1531 Query must start with ? and must not have # in it. It is optional. 1532 Anchor must start with # and can have anything else in it to end of string. It is optional. 1533 */ 1534 1535 this = Uri.init; // reset all state 1536 1537 // empty uri = nothing special 1538 if(uri.length == 0) { 1539 return; 1540 } 1541 1542 size_t idx; 1543 1544 scheme_loop: foreach(char c; uri[idx .. $]) { 1545 switch(c) { 1546 case ':': 1547 case '/': 1548 case '?': 1549 case '#': 1550 break scheme_loop; 1551 default: 1552 } 1553 idx++; 1554 } 1555 1556 if(idx == 0 && uri[idx] == ':') { 1557 // this is actually a path! we skip way ahead 1558 goto path_loop; 1559 } 1560 1561 if(idx == uri.length) { 1562 // the whole thing is a path, apparently 1563 path = uri; 1564 return; 1565 } 1566 1567 if(idx > 0 && uri[idx] == ':') { 1568 scheme = uri[0 .. idx]; 1569 idx++; 1570 } else { 1571 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 1572 idx = 0; 1573 } 1574 1575 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 1576 // we have an authority.... 1577 idx += 2; 1578 1579 auto authority_start = idx; 1580 authority_loop: foreach(char c; uri[idx .. $]) { 1581 switch(c) { 1582 case '/': 1583 case '?': 1584 case '#': 1585 break authority_loop; 1586 default: 1587 } 1588 idx++; 1589 } 1590 1591 auto authority = uri[authority_start .. idx]; 1592 1593 auto idx2 = authority.indexOf("@"); 1594 if(idx2 != -1) { 1595 userinfo = authority[0 .. idx2]; 1596 authority = authority[idx2 + 1 .. $]; 1597 } 1598 1599 auto remaining = authority; 1600 1601 while(remaining.length) { 1602 auto idx3 = remaining.indexOf(","); 1603 if(idx3 != -1) { 1604 authority = remaining[0 .. idx3]; 1605 remaining = remaining[idx3 + 1 .. $]; 1606 } else { 1607 authority = remaining; 1608 remaining = null; 1609 } 1610 1611 string host; 1612 int port; 1613 1614 idx2 = authority.indexOf(":"); 1615 if(idx2 == -1) { 1616 port = 0; // 0 means not specified; we should use the default for the scheme 1617 host = authority; 1618 } else { 1619 host = authority[0 .. idx2]; 1620 port = to!int(authority[idx2 + 1 .. $]); 1621 } 1622 1623 hosts ~= UriHost(host, port); 1624 } 1625 } 1626 1627 path_loop: 1628 auto path_start = idx; 1629 1630 foreach(char c; uri[idx .. $]) { 1631 if(c == '?' || c == '#') 1632 break; 1633 idx++; 1634 } 1635 1636 path = uri[path_start .. idx]; 1637 1638 if(idx == uri.length) 1639 return; // nothing more to examine... 1640 1641 if(uri[idx] == '?') { 1642 idx++; 1643 auto query_start = idx; 1644 foreach(char c; uri[idx .. $]) { 1645 if(c == '#') 1646 break; 1647 idx++; 1648 } 1649 query = uri[query_start .. idx]; 1650 } 1651 1652 if(idx < uri.length && uri[idx] == '#') { 1653 idx++; 1654 fragment = uri[idx .. $]; 1655 } 1656 1657 // uriInvalidated = false; 1658 } 1659 } 1660 /* end */ 1661 1662 1663 version(linux) 1664 @trusted 1665 extern(C) 1666 int getentropy(scope void*, size_t); 1667 else version(Windows) { 1668 import core.sys.windows.windows; 1669 import core.sys.windows.ntdef; 1670 enum STATUS_SUCCESS = 0; 1671 pragma(lib, "bcrypt"); 1672 @trusted 1673 extern(Windows) 1674 NTSTATUS BCryptGenRandom(scope void*, PUCHAR, ULONG, ULONG); 1675 } else static assert(0); 1676 1677 1678 1679 1680 // copy pasted from vibe.d; https://raw.githubusercontent.com/vibe-d/vibe.d/master/mongodb/vibe/db/mongo/sasl.d 1681 1682 /* 1683 SASL authentication functions 1684 1685 Copyright: © 2012-2016 Nicolas Gurrola 1686 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 1687 Authors: Nicolas Gurrola 1688 */ 1689 1690 import std.algorithm; 1691 import std.base64; 1692 import std.conv; 1693 import std.digest.hmac; 1694 import std.digest.sha; 1695 import std.exception; 1696 import std.format; 1697 import std.string; 1698 import std.traits; 1699 import std.utf; 1700 1701 @safe: 1702 1703 package struct ScramState 1704 { 1705 @safe: 1706 1707 private string m_firstMessageBare; 1708 private string m_nonce; 1709 private DigestType!SHA1 m_saltedPassword; 1710 private string m_authMessage; 1711 1712 string createInitialRequest(string user) 1713 { 1714 ubyte[18] randomBytes; 1715 version(linux) { 1716 if(getentropy(&(randomBytes[0]), randomBytes.length) != 0) 1717 throw new Exception("get random failure"); 1718 } else version(Windows) { 1719 if(BCryptGenRandom(null, &(randomBytes[0]), randomBytes.length, 2 /*BCRYPT_USE_SYSTEM_PREFERRED_RNG */) != STATUS_SUCCESS) 1720 throw new Exception("get random failure"); 1721 } else static assert(0); 1722 1723 m_nonce = Base64.encode(randomBytes); 1724 1725 m_firstMessageBare = format("n=%s,r=%s", escapeUsername(user), m_nonce); 1726 return format("n,,%s", m_firstMessageBare); 1727 } 1728 1729 version (unittest) private string createInitialRequestWithFixedNonce(string user, string nonce) 1730 { 1731 m_nonce = nonce; 1732 1733 m_firstMessageBare = format("n=%s,r=%s", escapeUsername(user), m_nonce); 1734 return format("n,,%s", m_firstMessageBare); 1735 } 1736 1737 // MongoDB drivers require 4096 min iterations https://github.com/mongodb/specifications/blob/59390a7ab2d5c8f9c29b8af1775ff25915c44036/source/auth/auth.rst#scram-sha-1 1738 string update(string password, string challenge, int minIterations = 4096) 1739 { 1740 string serverFirstMessage = challenge; 1741 1742 string next = challenge.find(','); 1743 if (challenge.length < 2 || challenge[0 .. 2] != "r=" || next.length < 3 || next[1 .. 3] != "s=") 1744 throw new Exception("Invalid server challenge format: " ~ challenge); 1745 string serverNonce = challenge[2 .. $ - next.length]; 1746 challenge = next[3 .. $]; 1747 next = challenge.find(','); 1748 ubyte[] salt = Base64.decode(challenge[0 .. $ - next.length]); 1749 1750 if (next.length < 3 || next[1 .. 3] != "i=") 1751 throw new Exception("Invalid server challenge format"); 1752 int iterations = next[3 .. $].to!int(); 1753 1754 if (iterations < minIterations) 1755 throw new Exception("Server must request at least " ~ minIterations.to!string ~ " iterations"); 1756 1757 if (serverNonce[0 .. m_nonce.length] != m_nonce) 1758 throw new Exception("Invalid server nonce received"); 1759 string finalMessage = format("c=biws,r=%s", serverNonce); 1760 1761 m_saltedPassword = pbkdf2(password.representation, salt, iterations); 1762 m_authMessage = format("%s,%s,%s", m_firstMessageBare, serverFirstMessage, finalMessage); 1763 1764 auto proof = getClientProof(m_saltedPassword, m_authMessage); 1765 return format("%s,p=%s", finalMessage, Base64.encode(proof)); 1766 } 1767 1768 string finalize(string challenge) 1769 { 1770 if (challenge.length < 2 || challenge[0 .. 2] != "v=") 1771 { 1772 throw new Exception("Invalid server signature format"); 1773 } 1774 if (!verifyServerSignature(Base64.decode(challenge[2 .. $]), m_saltedPassword, m_authMessage)) 1775 { 1776 throw new Exception("Invalid server signature"); 1777 } 1778 return null; 1779 } 1780 1781 private static string escapeUsername(string user) 1782 { 1783 char[] buffer; 1784 foreach (i, dchar ch; user) 1785 { 1786 if (ch == ',' || ch == '=') { 1787 if (!buffer) { 1788 buffer.reserve(user.length + 2); 1789 buffer ~= user[0 .. i]; 1790 } 1791 if (ch == ',') 1792 buffer ~= "=2C"; 1793 else 1794 buffer ~= "=3D"; 1795 } else if (buffer) 1796 encode(buffer, ch); 1797 } 1798 return buffer ? () @trusted { return assumeUnique(buffer); } () : user; 1799 } 1800 1801 unittest 1802 { 1803 string user = "user"; 1804 assert(escapeUsername(user) == user); 1805 assert(escapeUsername(user) is user); 1806 assert(escapeUsername("user,1") == "user=2C1"); 1807 assert(escapeUsername("user=1") == "user=3D1"); 1808 assert(escapeUsername("u,=ser1") == "u=2C=3Dser1"); 1809 assert(escapeUsername("u=se=r1") == "u=3Dse=3Dr1"); 1810 } 1811 1812 private static auto getClientProof(DigestType!SHA1 saltedPassword, string authMessage) 1813 { 1814 auto clientKey = () @trusted { return hmac!SHA1("Client Key".representation, saltedPassword); } (); 1815 auto storedKey = sha1Of(clientKey); 1816 auto clientSignature = () @trusted { return hmac!SHA1(authMessage.representation, storedKey); } (); 1817 1818 foreach (i; 0 .. clientKey.length) 1819 { 1820 clientKey[i] = clientKey[i] ^ clientSignature[i]; 1821 } 1822 return clientKey; 1823 } 1824 1825 private static bool verifyServerSignature(ubyte[] signature, DigestType!SHA1 saltedPassword, string authMessage) 1826 @trusted { 1827 auto serverKey = hmac!SHA1("Server Key".representation, saltedPassword); 1828 auto serverSignature = hmac!SHA1(authMessage.representation, serverKey); 1829 return serverSignature == signature; 1830 } 1831 } 1832 1833 private DigestType!SHA1 pbkdf2(const ubyte[] password, const ubyte[] salt, int iterations) 1834 { 1835 import std.bitmanip; 1836 1837 ubyte[4] intBytes = [0, 0, 0, 1]; 1838 auto last = () @trusted { return hmac!SHA1(salt, intBytes[], password); } (); 1839 static assert(isStaticArray!(typeof(last)), 1840 "Code is written so that the hash array is expected to be placed on the stack"); 1841 auto current = last; 1842 foreach (i; 1 .. iterations) 1843 { 1844 last = () @trusted { return hmac!SHA1(last[], password); } (); 1845 foreach (j; 0 .. current.length) 1846 { 1847 current[j] = current[j] ^ last[j]; 1848 } 1849 } 1850 return current; 1851 } 1852 1853 unittest { 1854 ScramState state; 1855 assert(state.createInitialRequestWithFixedNonce("user", "fyko+d2lbbFgONRv9qkxdawL") 1856 == "n,,n=user,r=fyko+d2lbbFgONRv9qkxdawL"); 1857 auto last = state.update(makeDigest("user", "pencil"), 1858 "r=fyko+d2lbbFgONRv9qkxdawLHo+Vgk7qvUOKUwuWLIWg4l/9SraGMHEE,s=rQ9ZY3MntBeuP3E1TDVC4w==,i=10000"); 1859 assert(last == "c=biws,r=fyko+d2lbbFgONRv9qkxdawLHo+Vgk7qvUOKUwuWLIWg4l/9SraGMHEE,p=MC2T8BvbmWRckDw8oWl5IVghwCY=", 1860 last); 1861 last = state.finalize("v=UMWeI25JD1yNYZRMpZ4VHvhZ9e0="); 1862 assert(last == "", last); 1863 } 1864 1865 string makeDigest(string username, string password) { 1866 import std.digest.md; 1867 return md5Of(username ~ ":mongo:" ~ password).toHexString().idup.toLower(); 1868 }