1228063Sbapt// Written in the D programming language. 295060Sjmallett 395060Sjmallett/** 41590SrgrimesNetworking client functionality as provided by $(HTTP _curl.haxx.se/libcurl, 51590Srgrimeslibcurl). The libcurl library must be installed on the system in order to use 61590Srgrimesthis module. 71590Srgrimes 81590Srgrimes$(SCRIPT inhibitQuickIndex = 1;) 91590Srgrimes 101590Srgrimes$(DIVC quickindex, 111590Srgrimes$(BOOKTABLE , 121590Srgrimes$(TR $(TH Category) $(TH Functions) 131590Srgrimes) 141590Srgrimes$(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get) 151590Srgrimes$(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace) 161590Srgrimes$(MYREF connect) $(MYREF byLine) $(MYREF byChunk) 171590Srgrimes$(MYREF byLineAsync) $(MYREF byChunkAsync) ) 181590Srgrimes) 19228063Sbapt$(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF 201590SrgrimesSMTP) ) 211590Srgrimes) 221590Srgrimes) 231590Srgrimes) 241590Srgrimes 251590SrgrimesNote: 261590SrgrimesYou may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"]) 271590Srgrimesto your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB). 281590Srgrimes 291590SrgrimesWindows x86 note: 301590SrgrimesA DMD compatible libcurl static library can be downloaded from the dlang.org 311590Srgrimes$(LINK2 http://dlang.org/download.html, download page). 321590Srgrimes 331590SrgrimesCompared to using libcurl directly this module allows simpler client code for 341590Srgrimescommon uses, requires no unsafe operations, and integrates better with the rest 351590Srgrimesof the language. Futhermore it provides <a href="std_range.html">$(D range)</a> 3695060Sjmallettaccess to protocols supported by libcurl both synchronously and asynchronously. 371590Srgrimes 381590SrgrimesA high level and a low level API are available. The high level API is built 391590Srgrimesentirely on top of the low level one. 401590Srgrimes 411590SrgrimesThe high level API is for commonly used functionality such as HTTP/FTP get. The 421590Srgrimes$(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous <a 431590Srgrimeshref="std_range.html">$(D ranges)</a> that performs the request in another 441590Srgrimesthread while handling a line/chunk in the current thread. 451590Srgrimes 461590SrgrimesThe low level API allows for streaming and other advanced features. 471590Srgrimes 48100014Sjmallett$(BOOKTABLE Cheat Sheet, 49100014Sjmallett$(TR $(TH Function Name) $(TH Description) 50100014Sjmallett) 511590Srgrimes$(LEADINGROW High level) 521590Srgrimes$(TR $(TDNW $(LREF download)) $(TD $(D 531590Srgrimesdownload("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file")) 541590Srgrimesdownloads file from URL to file system.) 551590Srgrimes) 561590Srgrimes$(TR $(TDNW $(LREF upload)) $(TD $(D 57upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");) 58uploads file from file system to URL.) 59) 60$(TR $(TDNW $(LREF get)) $(TD $(D 61get("dlang.org")) returns a char[] containing the dlang.org web page.) 62) 63$(TR $(TDNW $(LREF put)) $(TD $(D 64put("dlang.org", "Hi")) returns a char[] containing 65the dlang.org web page. after a HTTP PUT of "hi") 66) 67$(TR $(TDNW $(LREF post)) $(TD $(D 68post("dlang.org", "Hi")) returns a char[] containing 69the dlang.org web page. after a HTTP POST of "hi") 70) 71$(TR $(TDNW $(LREF byLine)) $(TD $(D 72byLine("dlang.org")) returns a range of char[] containing the 73dlang.org web page.) 74) 75$(TR $(TDNW $(LREF byChunk)) $(TD $(D 76byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the 77dlang.org web page.) 78) 79$(TR $(TDNW $(LREF byLineAsync)) $(TD $(D 80byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web 81 page asynchronously.) 82) 83$(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D 84byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the 85dlang.org web page asynchronously.) 86) 87$(LEADINGROW Low level 88) 89$(TR $(TDNW $(LREF HTTP)) $(TD $(D HTTP) struct for advanced usage)) 90$(TR $(TDNW $(LREF FTP)) $(TD $(D FTP) struct for advanced usage)) 91$(TR $(TDNW $(LREF SMTP)) $(TD $(D SMTP) struct for advanced usage)) 92) 93 94 95Example: 96--- 97import std.net.curl, std.stdio; 98 99// Return a char[] containing the content specified by a URL 100auto content = get("dlang.org"); 101 102// Post data and return a char[] containing the content specified by a URL 103auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]); 104 105// Get content of file from ftp server 106auto content = get("ftp.digitalmars.com/sieve.ds"); 107 108// Post and print out content line by line. The request is done in another thread. 109foreach (line; byLineAsync("dlang.org", "Post data")) 110 writeln(line); 111 112// Get using a line range and proxy settings 113auto client = HTTP(); 114client.proxy = "1.2.3.4"; 115foreach (line; byLine("dlang.org", client)) 116 writeln(line); 117--- 118 119For more control than the high level functions provide, use the low level API: 120 121Example: 122--- 123import std.net.curl, std.stdio; 124 125// GET with custom data receivers 126auto http = HTTP("dlang.org"); 127http.onReceiveHeader = 128 (in char[] key, in char[] value) { writeln(key, ": ", value); }; 129http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 130http.perform(); 131--- 132 133First, an instance of the reference-counted HTTP struct is created. Then the 134custom delegates are set. These will be called whenever the HTTP instance 135receives a header and a data buffer, respectively. In this simple example, the 136headers are written to stdout and the data is ignored. If the request should be 137stopped before it has finished then return something less than data.length from 138the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more 139information. Finally the HTTP request is effected by calling perform(), which is 140synchronous. 141 142Source: $(PHOBOSSRC std/net/_curl.d) 143 144Copyright: Copyright Jonas Drewsen 2011-2012 145License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 146Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao. 147 148Credits: The functionally is based on $(HTTP _curl.haxx.se/libcurl, libcurl). 149 LibCurl is licensed under an MIT/X derivative license. 150*/ 151/* 152 Copyright Jonas Drewsen 2011 - 2012. 153Distributed under the Boost Software License, Version 1.0. 154 (See accompanying file LICENSE_1_0.txt or copy at 155 http://www.boost.org/LICENSE_1_0.txt) 156*/ 157module std.net.curl; 158 159import core.thread; 160import etc.c.curl; 161import std.concurrency; 162import std.encoding; 163import std.exception; 164import std.meta; 165import std.range.primitives; 166import std.socket : InternetAddress; 167import std.traits; 168import std.typecons; 169 170import std.internal.cstring; 171 172public import etc.c.curl : CurlOption; 173 174version (unittest) 175{ 176 // Run unit test with the PHOBOS_TEST_ALLOW_NET=1 set in order to 177 // allow net traffic 178 import std.range; 179 import std.stdio; 180 181 import std.socket : Address, INADDR_LOOPBACK, Socket, SocketShutdown, TcpSocket; 182 183 private struct TestServer 184 { 185 string addr() { return _addr; } 186 187 void handle(void function(Socket s) dg) 188 { 189 tid.send(dg); 190 } 191 192 private: 193 string _addr; 194 Tid tid; 195 TcpSocket sock; 196 197 static void loop(shared TcpSocket listener) 198 { 199 try while (true) 200 { 201 void function(Socket) handler = void; 202 try 203 handler = receiveOnly!(typeof(handler)); 204 catch (OwnerTerminated) 205 return; 206 handler((cast() listener).accept); 207 } 208 catch (Throwable e) 209 { 210 stderr.writeln(e); // Bugzilla 7018 211 } 212 } 213 } 214 215 private TestServer startServer() 216 { 217 tlsInit = true; 218 auto sock = new TcpSocket; 219 sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY)); 220 sock.listen(1); 221 auto addr = sock.localAddress.toString(); 222 auto tid = spawn(&TestServer.loop, cast(shared) sock); 223 return TestServer(addr, tid, sock); 224 } 225 226 __gshared TestServer server; 227 bool tlsInit; 228 229 private ref TestServer testServer() 230 { 231 return initOnce!server(startServer()); 232 } 233 234 static ~this() 235 { 236 // terminate server from a thread local dtor of the thread that started it, 237 // because thread_joinall is called before shared module dtors 238 if (tlsInit && server.sock) 239 { 240 server.sock.shutdown(SocketShutdown.RECEIVE); 241 server.sock.close(); 242 } 243 } 244 245 private struct Request(T) 246 { 247 string hdrs; 248 immutable(T)[] bdy; 249 } 250 251 private Request!T recvReq(T=char)(Socket s) 252 { 253 import std.algorithm.comparison : min; 254 import std.algorithm.searching : find, canFind; 255 import std.conv : to; 256 import std.regex : ctRegex, matchFirst; 257 258 ubyte[1024] tmp=void; 259 ubyte[] buf; 260 261 while (true) 262 { 263 auto nbytes = s.receive(tmp[]); 264 assert(nbytes >= 0); 265 266 immutable beg = buf.length > 3 ? buf.length - 3 : 0; 267 buf ~= tmp[0 .. nbytes]; 268 auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n"); 269 if (bdy.empty) 270 continue; 271 272 auto hdrs = cast(string) buf[0 .. $ - bdy.length]; 273 bdy.popFrontN(4); 274 // no support for chunked transfer-encoding 275 if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i"))) 276 { 277 import std.uni : asUpperCase; 278 if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE")) 279 s.send(httpContinue); 280 281 size_t remain = m.captures[1].to!size_t - bdy.length; 282 while (remain) 283 { 284 nbytes = s.receive(tmp[0 .. min(remain, $)]); 285 assert(nbytes >= 0); 286 buf ~= tmp[0 .. nbytes]; 287 remain -= nbytes; 288 } 289 } 290 else 291 { 292 assert(bdy.empty); 293 } 294 bdy = buf[hdrs.length + 4 .. $]; 295 return typeof(return)(hdrs, cast(immutable(T)[])bdy); 296 } 297 } 298 299 private string httpOK(string msg) 300 { 301 import std.conv : to; 302 303 return "HTTP/1.1 200 OK\r\n"~ 304 "Content-Type: text/plain\r\n"~ 305 "Content-Length: "~msg.length.to!string~"\r\n"~ 306 "\r\n"~ 307 msg; 308 } 309 310 private string httpOK() 311 { 312 return "HTTP/1.1 200 OK\r\n"~ 313 "Content-Length: 0\r\n"~ 314 "\r\n"; 315 } 316 317 private string httpNotFound() 318 { 319 return "HTTP/1.1 404 Not Found\r\n"~ 320 "Content-Length: 0\r\n"~ 321 "\r\n"; 322 } 323 324 private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n"; 325} 326version (StdDdoc) import std.stdio; 327 328// Default data timeout for Protocols 329private enum _defaultDataTimeout = dur!"minutes"(2); 330 331/** 332Macros: 333 334CALLBACK_PARAMS = $(TABLE , 335 $(DDOC_PARAM_ROW 336 $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal)) 337 $(DDOC_PARAM_DESC total bytes to download) 338 ) 339 $(DDOC_PARAM_ROW 340 $(DDOC_PARAM_ID $(DDOC_PARAM dlNow)) 341 $(DDOC_PARAM_DESC currently downloaded bytes) 342 ) 343 $(DDOC_PARAM_ROW 344 $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal)) 345 $(DDOC_PARAM_DESC total bytes to upload) 346 ) 347 $(DDOC_PARAM_ROW 348 $(DDOC_PARAM_ID $(DDOC_PARAM ulNow)) 349 $(DDOC_PARAM_DESC currently uploaded bytes) 350 ) 351) 352*/ 353 354/** Connection type used when the URL should be used to auto detect the protocol. 355 * 356 * This struct is used as placeholder for the connection parameter when calling 357 * the high level API and the connection type (HTTP/FTP) should be guessed by 358 * inspecting the URL parameter. 359 * 360 * The rules for guessing the protocol are: 361 * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed. 362 * 2, HTTP connection otherwise. 363 * 364 * Example: 365 * --- 366 * import std.net.curl; 367 * // Two requests below will do the same. 368 * string content; 369 * 370 * // Explicit connection provided 371 * content = get!HTTP("dlang.org"); 372 * 373 * // Guess connection type by looking at the URL 374 * content = get!AutoProtocol("ftp://foo.com/file"); 375 * // and since AutoProtocol is default this is the same as 376 * content = get("ftp://foo.com/file"); 377 * // and will end up detecting FTP from the url and be the same as 378 * content = get!FTP("ftp://foo.com/file"); 379 * --- 380 */ 381struct AutoProtocol { } 382 383// Returns true if the url points to an FTP resource 384private bool isFTPUrl(const(char)[] url) 385{ 386 import std.algorithm.searching : startsWith; 387 import std.uni : toLower; 388 389 return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0; 390} 391 392// Is true if the Conn type is a valid Curl Connection type. 393private template isCurlConn(Conn) 394{ 395 enum auto isCurlConn = is(Conn : HTTP) || 396 is(Conn : FTP) || is(Conn : AutoProtocol); 397} 398 399/** HTTP/FTP download to local file system. 400 * 401 * Params: 402 * url = resource to download 403 * saveToPath = path to store the downloaded content on local disk 404 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 405 * guess connection type and create a new instance for this call only. 406 * 407 * Example: 408 * ---- 409 * import std.net.curl; 410 * download("d-lang.appspot.com/testUrl2", "/tmp/downloaded-http-file"); 411 * ---- 412 */ 413void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn()) 414if (isCurlConn!Conn) 415{ 416 static if (is(Conn : HTTP) || is(Conn : FTP)) 417 { 418 import std.stdio : File; 419 conn.url = url; 420 auto f = File(saveToPath, "wb"); 421 conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; }; 422 conn.perform(); 423 } 424 else 425 { 426 if (isFTPUrl(url)) 427 return download!FTP(url, saveToPath, FTP()); 428 else 429 return download!HTTP(url, saveToPath, HTTP()); 430 } 431} 432 433@system unittest 434{ 435 import std.algorithm.searching : canFind; 436 static import std.file; 437 438 foreach (host; [testServer.addr, "http://"~testServer.addr]) 439 { 440 testServer.handle((s) { 441 assert(s.recvReq.hdrs.canFind("GET /")); 442 s.send(httpOK("Hello world")); 443 }); 444 auto fn = std.file.deleteme; 445 scope (exit) 446 { 447 if (std.file.exists(fn)) 448 std.file.remove(fn); 449 } 450 download(host, fn); 451 assert(std.file.readText(fn) == "Hello world"); 452 } 453} 454 455/** Upload file from local files system using the HTTP or FTP protocol. 456 * 457 * Params: 458 * loadFromPath = path load data from local disk. 459 * url = resource to upload to 460 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 461 * guess connection type and create a new instance for this call only. 462 * 463 * Example: 464 * ---- 465 * import std.net.curl; 466 * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds"); 467 * upload("/tmp/downloaded-http-file", "d-lang.appspot.com/testUrl2"); 468 * ---- 469 */ 470void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn()) 471if (isCurlConn!Conn) 472{ 473 static if (is(Conn : HTTP)) 474 { 475 conn.url = url; 476 conn.method = HTTP.Method.put; 477 } 478 else static if (is(Conn : FTP)) 479 { 480 conn.url = url; 481 conn.handle.set(CurlOption.upload, 1L); 482 } 483 else 484 { 485 if (isFTPUrl(url)) 486 return upload!FTP(loadFromPath, url, FTP()); 487 else 488 return upload!HTTP(loadFromPath, url, HTTP()); 489 } 490 491 static if (is(Conn : HTTP) || is(Conn : FTP)) 492 { 493 import std.stdio : File; 494 auto f = File(loadFromPath, "rb"); 495 conn.onSend = buf => f.rawRead(buf).length; 496 immutable sz = f.size; 497 if (sz != ulong.max) 498 conn.contentLength = sz; 499 conn.perform(); 500 } 501} 502 503@system unittest 504{ 505 import std.algorithm.searching : canFind; 506 static import std.file; 507 508 foreach (host; [testServer.addr, "http://"~testServer.addr]) 509 { 510 auto fn = std.file.deleteme; 511 scope (exit) 512 { 513 if (std.file.exists(fn)) 514 std.file.remove(fn); 515 } 516 std.file.write(fn, "upload data\n"); 517 testServer.handle((s) { 518 auto req = s.recvReq; 519 assert(req.hdrs.canFind("PUT /path")); 520 assert(req.bdy.canFind("upload data")); 521 s.send(httpOK()); 522 }); 523 upload(fn, host ~ "/path"); 524 } 525} 526 527/** HTTP/FTP get content. 528 * 529 * Params: 530 * url = resource to get 531 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 532 * guess connection type and create a new instance for this call only. 533 * 534 * The template parameter $(D T) specifies the type to return. Possible values 535 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking 536 * for $(D char), content will be converted from the connection character set 537 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 538 * by default) to UTF-8. 539 * 540 * Example: 541 * ---- 542 * import std.net.curl; 543 * auto content = get("d-lang.appspot.com/testUrl2"); 544 * ---- 545 * 546 * Returns: 547 * A T[] range containing the content of the resource pointed to by the URL. 548 * 549 * Throws: 550 * 551 * $(D CurlException) on error. 552 * 553 * See_Also: $(LREF HTTP.Method) 554 */ 555T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn()) 556if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 557{ 558 static if (is(Conn : HTTP)) 559 { 560 conn.method = HTTP.Method.get; 561 return _basicHTTP!(T)(url, "", conn); 562 563 } 564 else static if (is(Conn : FTP)) 565 { 566 return _basicFTP!(T)(url, "", conn); 567 } 568 else 569 { 570 if (isFTPUrl(url)) 571 return get!(FTP,T)(url, FTP()); 572 else 573 return get!(HTTP,T)(url, HTTP()); 574 } 575} 576 577@system unittest 578{ 579 import std.algorithm.searching : canFind; 580 581 foreach (host; [testServer.addr, "http://"~testServer.addr]) 582 { 583 testServer.handle((s) { 584 assert(s.recvReq.hdrs.canFind("GET /path")); 585 s.send(httpOK("GETRESPONSE")); 586 }); 587 auto res = get(host ~ "/path"); 588 assert(res == "GETRESPONSE"); 589 } 590} 591 592 593/** HTTP post content. 594 * 595 * Params: 596 * url = resource to post to 597 * postDict = data to send as the body of the request. An associative array 598 * of $(D string) is accepted and will be encoded using 599 * www-form-urlencoding 600 * postData = data to send as the body of the request. An array 601 * of an arbitrary type is accepted and will be cast to ubyte[] 602 * before sending it. 603 * conn = HTTP connection to use 604 * T = The template parameter $(D T) specifies the type to return. Possible values 605 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking 606 * for $(D char), content will be converted from the connection character set 607 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 608 * by default) to UTF-8. 609 * 610 * Examples: 611 * ---- 612 * import std.net.curl; 613 * 614 * auto content1 = post("d-lang.appspot.com/testUrl2", ["name1" : "value1", "name2" : "value2"]); 615 * auto content2 = post("d-lang.appspot.com/testUrl2", [1,2,3,4]); 616 * ---- 617 * 618 * Returns: 619 * A T[] range containing the content of the resource pointed to by the URL. 620 * 621 * See_Also: $(LREF HTTP.Method) 622 */ 623T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP()) 624if (is(T == char) || is(T == ubyte)) 625{ 626 conn.method = HTTP.Method.post; 627 return _basicHTTP!(T)(url, postData, conn); 628} 629 630@system unittest 631{ 632 import std.algorithm.searching : canFind; 633 634 foreach (host; [testServer.addr, "http://"~testServer.addr]) 635 { 636 testServer.handle((s) { 637 auto req = s.recvReq; 638 assert(req.hdrs.canFind("POST /path")); 639 assert(req.bdy.canFind("POSTBODY")); 640 s.send(httpOK("POSTRESPONSE")); 641 }); 642 auto res = post(host ~ "/path", "POSTBODY"); 643 assert(res == "POSTRESPONSE"); 644 } 645} 646 647@system unittest 648{ 649 import std.algorithm.searching : canFind; 650 651 auto data = new ubyte[](256); 652 foreach (i, ref ub; data) 653 ub = cast(ubyte) i; 654 655 testServer.handle((s) { 656 auto req = s.recvReq!ubyte; 657 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 658 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 659 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 660 }); 661 auto res = post!ubyte(testServer.addr, data); 662 assert(res == cast(ubyte[])[17, 27, 35, 41]); 663} 664 665/// ditto 666T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP()) 667if (is(T == char) || is(T == ubyte)) 668{ 669 import std.uri : urlEncode; 670 671 return post(url, urlEncode(postDict), conn); 672} 673 674@system unittest 675{ 676 foreach (host; [testServer.addr, "http://" ~ testServer.addr]) 677 { 678 testServer.handle((s) { 679 auto req = s.recvReq!char; 680 s.send(httpOK(req.bdy)); 681 }); 682 auto res = post(host ~ "/path", ["name1" : "value1", "name2" : "value2"]); 683 assert(res == "name1=value1&name2=value2" || res == "name2=value2&name1=value1"); 684 } 685} 686 687/** HTTP/FTP put content. 688 * 689 * Params: 690 * url = resource to put 691 * putData = data to send as the body of the request. An array 692 * of an arbitrary type is accepted and will be cast to ubyte[] 693 * before sending it. 694 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 695 * guess connection type and create a new instance for this call only. 696 * 697 * The template parameter $(D T) specifies the type to return. Possible values 698 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking 699 * for $(D char), content will be converted from the connection character set 700 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 701 * by default) to UTF-8. 702 * 703 * Example: 704 * ---- 705 * import std.net.curl; 706 * auto content = put("d-lang.appspot.com/testUrl2", 707 * "Putting this data"); 708 * ---- 709 * 710 * Returns: 711 * A T[] range containing the content of the resource pointed to by the URL. 712 * 713 * See_Also: $(LREF HTTP.Method) 714 */ 715T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData, 716 Conn conn = Conn()) 717if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 718{ 719 static if (is(Conn : HTTP)) 720 { 721 conn.method = HTTP.Method.put; 722 return _basicHTTP!(T)(url, putData, conn); 723 } 724 else static if (is(Conn : FTP)) 725 { 726 return _basicFTP!(T)(url, putData, conn); 727 } 728 else 729 { 730 if (isFTPUrl(url)) 731 return put!(FTP,T)(url, putData, FTP()); 732 else 733 return put!(HTTP,T)(url, putData, HTTP()); 734 } 735} 736 737@system unittest 738{ 739 import std.algorithm.searching : canFind; 740 741 foreach (host; [testServer.addr, "http://"~testServer.addr]) 742 { 743 testServer.handle((s) { 744 auto req = s.recvReq; 745 assert(req.hdrs.canFind("PUT /path")); 746 assert(req.bdy.canFind("PUTBODY")); 747 s.send(httpOK("PUTRESPONSE")); 748 }); 749 auto res = put(host ~ "/path", "PUTBODY"); 750 assert(res == "PUTRESPONSE"); 751 } 752} 753 754 755/** HTTP/FTP delete content. 756 * 757 * Params: 758 * url = resource to delete 759 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 760 * guess connection type and create a new instance for this call only. 761 * 762 * Example: 763 * ---- 764 * import std.net.curl; 765 * del("d-lang.appspot.com/testUrl2"); 766 * ---- 767 * 768 * See_Also: $(LREF HTTP.Method) 769 */ 770void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn()) 771if (isCurlConn!Conn) 772{ 773 static if (is(Conn : HTTP)) 774 { 775 conn.method = HTTP.Method.del; 776 _basicHTTP!char(url, cast(void[]) null, conn); 777 } 778 else static if (is(Conn : FTP)) 779 { 780 import std.algorithm.searching : findSplitAfter; 781 import std.conv : text; 782 783 auto trimmed = url.findSplitAfter("ftp://")[1]; 784 auto t = trimmed.findSplitAfter("/"); 785 enum minDomainNameLength = 3; 786 enforce!CurlException(t[0].length > minDomainNameLength, 787 text("Invalid FTP URL for delete ", url)); 788 conn.url = t[0]; 789 790 enforce!CurlException(!t[1].empty, 791 text("No filename specified to delete for URL ", url)); 792 conn.addCommand("DELE " ~ t[1]); 793 conn.perform(); 794 } 795 else 796 { 797 if (isFTPUrl(url)) 798 return del!FTP(url, FTP()); 799 else 800 return del!HTTP(url, HTTP()); 801 } 802} 803 804@system unittest 805{ 806 import std.algorithm.searching : canFind; 807 808 foreach (host; [testServer.addr, "http://"~testServer.addr]) 809 { 810 testServer.handle((s) { 811 auto req = s.recvReq; 812 assert(req.hdrs.canFind("DELETE /path")); 813 s.send(httpOK()); 814 }); 815 del(host ~ "/path"); 816 } 817} 818 819 820/** HTTP options request. 821 * 822 * Params: 823 * url = resource make a option call to 824 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 825 * guess connection type and create a new instance for this call only. 826 * 827 * The template parameter $(D T) specifies the type to return. Possible values 828 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). 829 * 830 * Example: 831 * ---- 832 * import std.net.curl; 833 * auto http = HTTP(); 834 * options("d-lang.appspot.com/testUrl2", http); 835 * writeln("Allow set to " ~ http.responseHeaders["Allow"]); 836 * ---- 837 * 838 * Returns: 839 * A T[] range containing the options of the resource pointed to by the URL. 840 * 841 * See_Also: $(LREF HTTP.Method) 842 */ 843T[] options(T = char)(const(char)[] url, HTTP conn = HTTP()) 844if (is(T == char) || is(T == ubyte)) 845{ 846 conn.method = HTTP.Method.options; 847 return _basicHTTP!(T)(url, null, conn); 848} 849 850@system unittest 851{ 852 import std.algorithm.searching : canFind; 853 854 testServer.handle((s) { 855 auto req = s.recvReq; 856 assert(req.hdrs.canFind("OPTIONS /path")); 857 s.send(httpOK("OPTIONSRESPONSE")); 858 }); 859 auto res = options(testServer.addr ~ "/path"); 860 assert(res == "OPTIONSRESPONSE"); 861} 862 863 864/** HTTP trace request. 865 * 866 * Params: 867 * url = resource make a trace call to 868 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 869 * guess connection type and create a new instance for this call only. 870 * 871 * The template parameter $(D T) specifies the type to return. Possible values 872 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). 873 * 874 * Example: 875 * ---- 876 * import std.net.curl; 877 * trace("d-lang.appspot.com/testUrl1"); 878 * ---- 879 * 880 * Returns: 881 * A T[] range containing the trace info of the resource pointed to by the URL. 882 * 883 * See_Also: $(LREF HTTP.Method) 884 */ 885T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP()) 886if (is(T == char) || is(T == ubyte)) 887{ 888 conn.method = HTTP.Method.trace; 889 return _basicHTTP!(T)(url, cast(void[]) null, conn); 890} 891 892@system unittest 893{ 894 import std.algorithm.searching : canFind; 895 896 testServer.handle((s) { 897 auto req = s.recvReq; 898 assert(req.hdrs.canFind("TRACE /path")); 899 s.send(httpOK("TRACERESPONSE")); 900 }); 901 auto res = trace(testServer.addr ~ "/path"); 902 assert(res == "TRACERESPONSE"); 903} 904 905 906/** HTTP connect request. 907 * 908 * Params: 909 * url = resource make a connect to 910 * conn = HTTP connection to use 911 * 912 * The template parameter $(D T) specifies the type to return. Possible values 913 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). 914 * 915 * Example: 916 * ---- 917 * import std.net.curl; 918 * connect("d-lang.appspot.com/testUrl1"); 919 * ---- 920 * 921 * Returns: 922 * A T[] range containing the connect info of the resource pointed to by the URL. 923 * 924 * See_Also: $(LREF HTTP.Method) 925 */ 926T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP()) 927if (is(T == char) || is(T == ubyte)) 928{ 929 conn.method = HTTP.Method.connect; 930 return _basicHTTP!(T)(url, cast(void[]) null, conn); 931} 932 933@system unittest 934{ 935 import std.algorithm.searching : canFind; 936 937 testServer.handle((s) { 938 auto req = s.recvReq; 939 assert(req.hdrs.canFind("CONNECT /path")); 940 s.send(httpOK("CONNECTRESPONSE")); 941 }); 942 auto res = connect(testServer.addr ~ "/path"); 943 assert(res == "CONNECTRESPONSE"); 944} 945 946 947/** HTTP patch content. 948 * 949 * Params: 950 * url = resource to patch 951 * patchData = data to send as the body of the request. An array 952 * of an arbitrary type is accepted and will be cast to ubyte[] 953 * before sending it. 954 * conn = HTTP connection to use 955 * 956 * The template parameter $(D T) specifies the type to return. Possible values 957 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). 958 * 959 * Example: 960 * ---- 961 * auto http = HTTP(); 962 * http.addRequestHeader("Content-Type", "application/json"); 963 * auto content = patch("d-lang.appspot.com/testUrl2", `{"title": "Patched Title"}`, http); 964 * ---- 965 * 966 * Returns: 967 * A T[] range containing the content of the resource pointed to by the URL. 968 * 969 * See_Also: $(LREF HTTP.Method) 970 */ 971T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData, 972 HTTP conn = HTTP()) 973if (is(T == char) || is(T == ubyte)) 974{ 975 conn.method = HTTP.Method.patch; 976 return _basicHTTP!(T)(url, patchData, conn); 977} 978 979@system unittest 980{ 981 import std.algorithm.searching : canFind; 982 983 testServer.handle((s) { 984 auto req = s.recvReq; 985 assert(req.hdrs.canFind("PATCH /path")); 986 assert(req.bdy.canFind("PATCHBODY")); 987 s.send(httpOK("PATCHRESPONSE")); 988 }); 989 auto res = patch(testServer.addr ~ "/path", "PATCHBODY"); 990 assert(res == "PATCHRESPONSE"); 991} 992 993 994/* 995 * Helper function for the high level interface. 996 * 997 * It performs an HTTP request using the client which must have 998 * been setup correctly before calling this function. 999 */ 1000private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client) 1001{ 1002 import std.algorithm.comparison : min; 1003 import std.format : format; 1004 1005 immutable doSend = sendData !is null && 1006 (client.method == HTTP.Method.post || 1007 client.method == HTTP.Method.put || 1008 client.method == HTTP.Method.patch); 1009 1010 scope (exit) 1011 { 1012 client.onReceiveHeader = null; 1013 client.onReceiveStatusLine = null; 1014 client.onReceive = null; 1015 1016 if (doSend) 1017 { 1018 client.onSend = null; 1019 client.handle.onSeek = null; 1020 client.contentLength = 0; 1021 } 1022 } 1023 client.url = url; 1024 HTTP.StatusLine statusLine; 1025 import std.array : appender; 1026 auto content = appender!(ubyte[])(); 1027 client.onReceive = (ubyte[] data) 1028 { 1029 content ~= data; 1030 return data.length; 1031 }; 1032 1033 if (doSend) 1034 { 1035 client.contentLength = sendData.length; 1036 auto remainingData = sendData; 1037 client.onSend = delegate size_t(void[] buf) 1038 { 1039 size_t minLen = min(buf.length, remainingData.length); 1040 if (minLen == 0) return 0; 1041 buf[0 .. minLen] = remainingData[0 .. minLen]; 1042 remainingData = remainingData[minLen..$]; 1043 return minLen; 1044 }; 1045 client.handle.onSeek = delegate(long offset, CurlSeekPos mode) 1046 { 1047 switch (mode) 1048 { 1049 case CurlSeekPos.set: 1050 remainingData = sendData[cast(size_t) offset..$]; 1051 return CurlSeek.ok; 1052 default: 1053 // As of curl 7.18.0, libcurl will not pass 1054 // anything other than CurlSeekPos.set. 1055 return CurlSeek.cantseek; 1056 } 1057 }; 1058 } 1059 1060 client.onReceiveHeader = (in char[] key, 1061 in char[] value) 1062 { 1063 if (key == "content-length") 1064 { 1065 import std.conv : to; 1066 content.reserve(value.to!size_t); 1067 } 1068 }; 1069 client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; }; 1070 client.perform(); 1071 enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code, 1072 format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason))); 1073 1074 return _decodeContent!T(content.data, client.p.charset); 1075} 1076 1077@system unittest 1078{ 1079 import std.algorithm.searching : canFind; 1080 1081 testServer.handle((s) { 1082 auto req = s.recvReq; 1083 assert(req.hdrs.canFind("GET /path")); 1084 s.send(httpNotFound()); 1085 }); 1086 auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path")); 1087 assert(e.msg == "HTTP request returned status code 404 (Not Found)"); 1088 assert(e.status == 404); 1089} 1090 1091// Bugzilla 14760 - content length must be reset after post 1092@system unittest 1093{ 1094 import std.algorithm.searching : canFind; 1095 1096 testServer.handle((s) { 1097 auto req = s.recvReq; 1098 assert(req.hdrs.canFind("POST /")); 1099 assert(req.bdy.canFind("POSTBODY")); 1100 s.send(httpOK("POSTRESPONSE")); 1101 1102 req = s.recvReq; 1103 assert(req.hdrs.canFind("TRACE /")); 1104 assert(req.bdy.empty); 1105 s.blocking = false; 1106 ubyte[6] buf = void; 1107 assert(s.receive(buf[]) < 0); 1108 s.send(httpOK("TRACERESPONSE")); 1109 }); 1110 auto http = HTTP(); 1111 auto res = post(testServer.addr, "POSTBODY", http); 1112 assert(res == "POSTRESPONSE"); 1113 res = trace(testServer.addr, http); 1114 assert(res == "TRACERESPONSE"); 1115} 1116 1117@system unittest // charset detection and transcoding to T 1118{ 1119 testServer.handle((s) { 1120 s.send("HTTP/1.1 200 OK\r\n"~ 1121 "Content-Length: 4\r\n"~ 1122 "Content-Type: text/plain; charset=utf-8\r\n" ~ 1123 "\r\n" ~ 1124 "��bc"); 1125 }); 1126 auto client = HTTP(); 1127 auto result = _basicHTTP!char(testServer.addr, "", client); 1128 assert(result == "��bc"); 1129 1130 testServer.handle((s) { 1131 s.send("HTTP/1.1 200 OK\r\n"~ 1132 "Content-Length: 3\r\n"~ 1133 "Content-Type: text/plain; charset=iso-8859-1\r\n" ~ 1134 "\r\n" ~ 1135 0xE4 ~ "bc"); 1136 }); 1137 client = HTTP(); 1138 result = _basicHTTP!char(testServer.addr, "", client); 1139 assert(result == "��bc"); 1140} 1141 1142/* 1143 * Helper function for the high level interface. 1144 * 1145 * It performs an FTP request using the client which must have 1146 * been setup correctly before calling this function. 1147 */ 1148private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client) 1149{ 1150 import std.algorithm.comparison : min; 1151 1152 scope (exit) 1153 { 1154 client.onReceive = null; 1155 if (!sendData.empty) 1156 client.onSend = null; 1157 } 1158 1159 ubyte[] content; 1160 1161 if (client.encoding.empty) 1162 client.encoding = "ISO-8859-1"; 1163 1164 client.url = url; 1165 client.onReceive = (ubyte[] data) 1166 { 1167 content ~= data; 1168 return data.length; 1169 }; 1170 1171 if (!sendData.empty) 1172 { 1173 client.handle.set(CurlOption.upload, 1L); 1174 client.onSend = delegate size_t(void[] buf) 1175 { 1176 size_t minLen = min(buf.length, sendData.length); 1177 if (minLen == 0) return 0; 1178 buf[0 .. minLen] = sendData[0 .. minLen]; 1179 sendData = sendData[minLen..$]; 1180 return minLen; 1181 }; 1182 } 1183 1184 client.perform(); 1185 1186 return _decodeContent!T(content, client.encoding); 1187} 1188 1189/* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to 1190 * correct string format 1191 */ 1192private auto _decodeContent(T)(ubyte[] content, string encoding) 1193{ 1194 static if (is(T == ubyte)) 1195 { 1196 return content; 1197 } 1198 else 1199 { 1200 import std.format : format; 1201 1202 // Optimally just return the utf8 encoded content 1203 if (encoding == "UTF-8") 1204 return cast(char[])(content); 1205 1206 // The content has to be re-encoded to utf8 1207 auto scheme = EncodingScheme.create(encoding); 1208 enforce!CurlException(scheme !is null, 1209 format("Unknown encoding '%s'", encoding)); 1210 1211 auto strInfo = decodeString(content, scheme); 1212 enforce!CurlException(strInfo[0] != size_t.max, 1213 format("Invalid encoding sequence for encoding '%s'", 1214 encoding)); 1215 1216 return strInfo[1]; 1217 } 1218} 1219 1220alias KeepTerminator = Flag!"keepTerminator"; 1221/+ 1222struct ByLineBuffer(Char) 1223{ 1224 bool linePresent; 1225 bool EOF; 1226 Char[] buffer; 1227 ubyte[] decodeRemainder; 1228 1229 bool append(const(ubyte)[] data) 1230 { 1231 byLineBuffer ~= data; 1232 } 1233 1234 @property bool linePresent() 1235 { 1236 return byLinePresent; 1237 } 1238 1239 Char[] get() 1240 { 1241 if (!linePresent) 1242 { 1243 // Decode ubyte[] into Char[] until a Terminator is found. 1244 // If not Terminator is found and EOF is false then raise an 1245 // exception. 1246 } 1247 return byLineBuffer; 1248 } 1249 1250} 1251++/ 1252/** HTTP/FTP fetch content as a range of lines. 1253 * 1254 * A range of lines is returned when the request is complete. If the method or 1255 * other request properties is to be customized then set the $(D conn) parameter 1256 * with a HTTP/FTP instance that has these properties set. 1257 * 1258 * Example: 1259 * ---- 1260 * import std.net.curl, std.stdio; 1261 * foreach (line; byLine("dlang.org")) 1262 * writeln(line); 1263 * ---- 1264 * 1265 * Params: 1266 * url = The url to receive content from 1267 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be 1268 * returned as part of the lines in the range. 1269 * terminator = The character that terminates a line 1270 * conn = The connection to use e.g. HTTP or FTP. 1271 * 1272 * Returns: 1273 * A range of Char[] with the content of the resource pointer to by the URL 1274 */ 1275auto byLine(Conn = AutoProtocol, Terminator = char, Char = char) 1276 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1277 Terminator terminator = '\n', Conn conn = Conn()) 1278if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1279{ 1280 static struct SyncLineInputRange 1281 { 1282 1283 private Char[] lines; 1284 private Char[] current; 1285 private bool currentValid; 1286 private bool keepTerminator; 1287 private Terminator terminator; 1288 1289 this(Char[] lines, bool kt, Terminator terminator) 1290 { 1291 this.lines = lines; 1292 this.keepTerminator = kt; 1293 this.terminator = terminator; 1294 currentValid = true; 1295 popFront(); 1296 } 1297 1298 @property @safe bool empty() 1299 { 1300 return !currentValid; 1301 } 1302 1303 @property @safe Char[] front() 1304 { 1305 enforce!CurlException(currentValid, "Cannot call front() on empty range"); 1306 return current; 1307 } 1308 1309 void popFront() 1310 { 1311 import std.algorithm.searching : findSplitAfter, findSplit; 1312 1313 enforce!CurlException(currentValid, "Cannot call popFront() on empty range"); 1314 if (lines.empty) 1315 { 1316 currentValid = false; 1317 return; 1318 } 1319 1320 if (keepTerminator) 1321 { 1322 auto r = findSplitAfter(lines, [ terminator ]); 1323 if (r[0].empty) 1324 { 1325 current = r[1]; 1326 lines = r[0]; 1327 } 1328 else 1329 { 1330 current = r[0]; 1331 lines = r[1]; 1332 } 1333 } 1334 else 1335 { 1336 auto r = findSplit(lines, [ terminator ]); 1337 current = r[0]; 1338 lines = r[2]; 1339 } 1340 } 1341 } 1342 1343 auto result = _getForRange!Char(url, conn); 1344 return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator); 1345} 1346 1347@system unittest 1348{ 1349 import std.algorithm.comparison : equal; 1350 1351 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1352 { 1353 testServer.handle((s) { 1354 auto req = s.recvReq; 1355 s.send(httpOK("Line1\nLine2\nLine3")); 1356 }); 1357 assert(byLine(host).equal(["Line1", "Line2", "Line3"])); 1358 } 1359} 1360 1361/** HTTP/FTP fetch content as a range of chunks. 1362 * 1363 * A range of chunks is returned when the request is complete. If the method or 1364 * other request properties is to be customized then set the $(D conn) parameter 1365 * with a HTTP/FTP instance that has these properties set. 1366 * 1367 * Example: 1368 * ---- 1369 * import std.net.curl, std.stdio; 1370 * foreach (chunk; byChunk("dlang.org", 100)) 1371 * writeln(chunk); // chunk is ubyte[100] 1372 * ---- 1373 * 1374 * Params: 1375 * url = The url to receive content from 1376 * chunkSize = The size of each chunk 1377 * conn = The connection to use e.g. HTTP or FTP. 1378 * 1379 * Returns: 1380 * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL 1381 */ 1382auto byChunk(Conn = AutoProtocol) 1383 (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn()) 1384if (isCurlConn!(Conn)) 1385{ 1386 static struct SyncChunkInputRange 1387 { 1388 private size_t chunkSize; 1389 private ubyte[] _bytes; 1390 private size_t offset; 1391 1392 this(ubyte[] bytes, size_t chunkSize) 1393 { 1394 this._bytes = bytes; 1395 this.chunkSize = chunkSize; 1396 } 1397 1398 @property @safe auto empty() 1399 { 1400 return offset == _bytes.length; 1401 } 1402 1403 @property ubyte[] front() 1404 { 1405 size_t nextOffset = offset + chunkSize; 1406 if (nextOffset > _bytes.length) nextOffset = _bytes.length; 1407 return _bytes[offset .. nextOffset]; 1408 } 1409 1410 @safe void popFront() 1411 { 1412 offset += chunkSize; 1413 if (offset > _bytes.length) offset = _bytes.length; 1414 } 1415 } 1416 1417 auto result = _getForRange!ubyte(url, conn); 1418 return SyncChunkInputRange(result, chunkSize); 1419} 1420 1421@system unittest 1422{ 1423 import std.algorithm.comparison : equal; 1424 1425 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1426 { 1427 testServer.handle((s) { 1428 auto req = s.recvReq; 1429 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1430 }); 1431 assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1432 } 1433} 1434 1435private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn) 1436{ 1437 static if (is(Conn : HTTP)) 1438 { 1439 conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method; 1440 return _basicHTTP!(T)(url, null, conn); 1441 } 1442 else static if (is(Conn : FTP)) 1443 { 1444 return _basicFTP!(T)(url, null, conn); 1445 } 1446 else 1447 { 1448 if (isFTPUrl(url)) 1449 return get!(FTP,T)(url, FTP()); 1450 else 1451 return get!(HTTP,T)(url, HTTP()); 1452 } 1453} 1454 1455/* 1456 Main thread part of the message passing protocol used for all async 1457 curl protocols. 1458 */ 1459private mixin template WorkerThreadProtocol(Unit, alias units) 1460{ 1461 @property bool empty() 1462 { 1463 tryEnsureUnits(); 1464 return state == State.done; 1465 } 1466 1467 @property Unit[] front() 1468 { 1469 import std.format : format; 1470 tryEnsureUnits(); 1471 assert(state == State.gotUnits, 1472 format("Expected %s but got $s", 1473 State.gotUnits, state)); 1474 return units; 1475 } 1476 1477 void popFront() 1478 { 1479 import std.format : format; 1480 tryEnsureUnits(); 1481 assert(state == State.gotUnits, 1482 format("Expected %s but got $s", 1483 State.gotUnits, state)); 1484 state = State.needUnits; 1485 // Send to worker thread for buffer reuse 1486 workerTid.send(cast(immutable(Unit)[]) units); 1487 units = null; 1488 } 1489 1490 /** Wait for duration or until data is available and return true if data is 1491 available 1492 */ 1493 bool wait(Duration d) 1494 { 1495 import std.datetime.stopwatch : StopWatch; 1496 1497 if (state == State.gotUnits) 1498 return true; 1499 1500 enum noDur = dur!"hnsecs"(0); 1501 StopWatch sw; 1502 sw.start(); 1503 while (state != State.gotUnits && d > noDur) 1504 { 1505 final switch (state) 1506 { 1507 case State.needUnits: 1508 receiveTimeout(d, 1509 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1510 { 1511 if (origin != workerTid) 1512 return false; 1513 units = cast(Unit[]) _data.data; 1514 state = State.gotUnits; 1515 return true; 1516 }, 1517 (Tid origin, CurlMessage!bool f) 1518 { 1519 if (origin != workerTid) 1520 return false; 1521 state = state.done; 1522 return true; 1523 } 1524 ); 1525 break; 1526 case State.gotUnits: return true; 1527 case State.done: 1528 return false; 1529 } 1530 d -= sw.peek(); 1531 sw.reset(); 1532 } 1533 return state == State.gotUnits; 1534 } 1535 1536 enum State 1537 { 1538 needUnits, 1539 gotUnits, 1540 done 1541 } 1542 State state; 1543 1544 void tryEnsureUnits() 1545 { 1546 while (true) 1547 { 1548 final switch (state) 1549 { 1550 case State.needUnits: 1551 receive( 1552 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1553 { 1554 if (origin != workerTid) 1555 return false; 1556 units = cast(Unit[]) _data.data; 1557 state = State.gotUnits; 1558 return true; 1559 }, 1560 (Tid origin, CurlMessage!bool f) 1561 { 1562 if (origin != workerTid) 1563 return false; 1564 state = state.done; 1565 return true; 1566 } 1567 ); 1568 break; 1569 case State.gotUnits: return; 1570 case State.done: 1571 return; 1572 } 1573 } 1574 } 1575} 1576 1577// @@@@BUG 15831@@@@ 1578// this should be inside byLineAsync 1579// Range that reads one line at a time asynchronously. 1580private static struct AsyncLineInputRange(Char) 1581{ 1582 private Char[] line; 1583 mixin WorkerThreadProtocol!(Char, line); 1584 1585 private Tid workerTid; 1586 private State running; 1587 1588 private this(Tid tid, size_t transmitBuffers, size_t bufferSize) 1589 { 1590 workerTid = tid; 1591 state = State.needUnits; 1592 1593 // Send buffers to other thread for it to use. Since no mechanism is in 1594 // place for moving ownership a cast to shared is done here and casted 1595 // back to non-shared in the receiving end. 1596 foreach (i ; 0 .. transmitBuffers) 1597 { 1598 auto arr = new Char[](bufferSize); 1599 workerTid.send(cast(immutable(Char[]))arr); 1600 } 1601 } 1602} 1603 1604/** HTTP/FTP fetch content as a range of lines asynchronously. 1605 * 1606 * A range of lines is returned immediately and the request that fetches the 1607 * lines is performed in another thread. If the method or other request 1608 * properties is to be customized then set the $(D conn) parameter with a 1609 * HTTP/FTP instance that has these properties set. 1610 * 1611 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP 1612 * requests. 1613 * 1614 * The background thread will buffer up to transmitBuffers number of lines 1615 * before it stops receiving data from network. When the main thread reads the 1616 * lines from the range it frees up buffers and allows for the background thread 1617 * to receive more data from the network. 1618 * 1619 * If no data is available and the main thread accesses the range it will block 1620 * until data becomes available. An exception to this is the $(D wait(Duration)) method on 1621 * the $(LREF AsyncLineInputRange). This method will wait at maximum for the 1622 * specified duration and return true if data is available. 1623 * 1624 * Example: 1625 * ---- 1626 * import std.net.curl, std.stdio; 1627 * // Get some pages in the background 1628 * auto range1 = byLineAsync("www.google.com"); 1629 * auto range2 = byLineAsync("www.wikipedia.org"); 1630 * foreach (line; byLineAsync("dlang.org")) 1631 * writeln(line); 1632 * 1633 * // Lines already fetched in the background and ready 1634 * foreach (line; range1) writeln(line); 1635 * foreach (line; range2) writeln(line); 1636 * ---- 1637 * 1638 * ---- 1639 * import std.net.curl, std.stdio; 1640 * // Get a line in a background thread and wait in 1641 * // main thread for 2 seconds for it to arrive. 1642 * auto range3 = byLineAsync("dlang.com"); 1643 * if (range3.wait(dur!"seconds"(2))) 1644 * writeln(range3.front); 1645 * else 1646 * writeln("No line received after 2 seconds!"); 1647 * ---- 1648 * 1649 * Params: 1650 * url = The url to receive content from 1651 * postData = Data to HTTP Post 1652 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be 1653 * returned as part of the lines in the range. 1654 * terminator = The character that terminates a line 1655 * transmitBuffers = The number of lines buffered asynchronously 1656 * conn = The connection to use e.g. HTTP or FTP. 1657 * 1658 * Returns: 1659 * A range of Char[] with the content of the resource pointer to by the 1660 * URL. 1661 */ 1662auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit) 1663 (const(char)[] url, const(PostUnit)[] postData, 1664 KeepTerminator keepTerminator = No.keepTerminator, 1665 Terminator terminator = '\n', 1666 size_t transmitBuffers = 10, Conn conn = Conn()) 1667if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1668{ 1669 static if (is(Conn : AutoProtocol)) 1670 { 1671 if (isFTPUrl(url)) 1672 return byLineAsync(url, postData, keepTerminator, 1673 terminator, transmitBuffers, FTP()); 1674 else 1675 return byLineAsync(url, postData, keepTerminator, 1676 terminator, transmitBuffers, HTTP()); 1677 } 1678 else 1679 { 1680 // 50 is just an arbitrary number for now 1681 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1682 auto tid = spawn(&_spawnAsync!(Conn, Char, Terminator)); 1683 tid.send(thisTid); 1684 tid.send(terminator); 1685 tid.send(keepTerminator == Yes.keepTerminator); 1686 1687 _asyncDuplicateConnection(url, conn, postData, tid); 1688 1689 return AsyncLineInputRange!Char(tid, transmitBuffers, 1690 Conn.defaultAsyncStringBufferSize); 1691 } 1692} 1693 1694/// ditto 1695auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char) 1696 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1697 Terminator terminator = '\n', 1698 size_t transmitBuffers = 10, Conn conn = Conn()) 1699{ 1700 static if (is(Conn : AutoProtocol)) 1701 { 1702 if (isFTPUrl(url)) 1703 return byLineAsync(url, cast(void[]) null, keepTerminator, 1704 terminator, transmitBuffers, FTP()); 1705 else 1706 return byLineAsync(url, cast(void[]) null, keepTerminator, 1707 terminator, transmitBuffers, HTTP()); 1708 } 1709 else 1710 { 1711 return byLineAsync(url, cast(void[]) null, keepTerminator, 1712 terminator, transmitBuffers, conn); 1713 } 1714} 1715 1716@system unittest 1717{ 1718 import std.algorithm.comparison : equal; 1719 1720 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1721 { 1722 testServer.handle((s) { 1723 auto req = s.recvReq; 1724 s.send(httpOK("Line1\nLine2\nLine3")); 1725 }); 1726 assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"])); 1727 } 1728} 1729 1730// @@@@BUG 15831@@@@ 1731// this should be inside byLineAsync 1732// Range that reads one chunk at a time asynchronously. 1733private static struct AsyncChunkInputRange 1734{ 1735 private ubyte[] chunk; 1736 mixin WorkerThreadProtocol!(ubyte, chunk); 1737 1738 private Tid workerTid; 1739 private State running; 1740 1741 private this(Tid tid, size_t transmitBuffers, size_t chunkSize) 1742 { 1743 workerTid = tid; 1744 state = State.needUnits; 1745 1746 // Send buffers to other thread for it to use. Since no mechanism is in 1747 // place for moving ownership a cast to shared is done here and a cast 1748 // back to non-shared in the receiving end. 1749 foreach (i ; 0 .. transmitBuffers) 1750 { 1751 ubyte[] arr = new ubyte[](chunkSize); 1752 workerTid.send(cast(immutable(ubyte[]))arr); 1753 } 1754 } 1755} 1756 1757/** HTTP/FTP fetch content as a range of chunks asynchronously. 1758 * 1759 * A range of chunks is returned immediately and the request that fetches the 1760 * chunks is performed in another thread. If the method or other request 1761 * properties is to be customized then set the $(D conn) parameter with a 1762 * HTTP/FTP instance that has these properties set. 1763 * 1764 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP 1765 * requests. 1766 * 1767 * The background thread will buffer up to transmitBuffers number of chunks 1768 * before is stops receiving data from network. When the main thread reads the 1769 * chunks from the range it frees up buffers and allows for the background 1770 * thread to receive more data from the network. 1771 * 1772 * If no data is available and the main thread access the range it will block 1773 * until data becomes available. An exception to this is the $(D wait(Duration)) 1774 * method on the $(LREF AsyncChunkInputRange). This method will wait at maximum for the specified 1775 * duration and return true if data is available. 1776 * 1777 * Example: 1778 * ---- 1779 * import std.net.curl, std.stdio; 1780 * // Get some pages in the background 1781 * auto range1 = byChunkAsync("www.google.com", 100); 1782 * auto range2 = byChunkAsync("www.wikipedia.org"); 1783 * foreach (chunk; byChunkAsync("dlang.org")) 1784 * writeln(chunk); // chunk is ubyte[100] 1785 * 1786 * // Chunks already fetched in the background and ready 1787 * foreach (chunk; range1) writeln(chunk); 1788 * foreach (chunk; range2) writeln(chunk); 1789 * ---- 1790 * 1791 * ---- 1792 * import std.net.curl, std.stdio; 1793 * // Get a line in a background thread and wait in 1794 * // main thread for 2 seconds for it to arrive. 1795 * auto range3 = byChunkAsync("dlang.com", 10); 1796 * if (range3.wait(dur!"seconds"(2))) 1797 * writeln(range3.front); 1798 * else 1799 * writeln("No chunk received after 2 seconds!"); 1800 * ---- 1801 * 1802 * Params: 1803 * url = The url to receive content from 1804 * postData = Data to HTTP Post 1805 * chunkSize = The size of the chunks 1806 * transmitBuffers = The number of chunks buffered asynchronously 1807 * conn = The connection to use e.g. HTTP or FTP. 1808 * 1809 * Returns: 1810 * A range of ubyte[chunkSize] with the content of the resource pointer to by 1811 * the URL. 1812 */ 1813auto byChunkAsync(Conn = AutoProtocol, PostUnit) 1814 (const(char)[] url, const(PostUnit)[] postData, 1815 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1816 Conn conn = Conn()) 1817if (isCurlConn!(Conn)) 1818{ 1819 static if (is(Conn : AutoProtocol)) 1820 { 1821 if (isFTPUrl(url)) 1822 return byChunkAsync(url, postData, chunkSize, 1823 transmitBuffers, FTP()); 1824 else 1825 return byChunkAsync(url, postData, chunkSize, 1826 transmitBuffers, HTTP()); 1827 } 1828 else 1829 { 1830 // 50 is just an arbitrary number for now 1831 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1832 auto tid = spawn(&_spawnAsync!(Conn, ubyte)); 1833 tid.send(thisTid); 1834 1835 _asyncDuplicateConnection(url, conn, postData, tid); 1836 1837 return AsyncChunkInputRange(tid, transmitBuffers, chunkSize); 1838 } 1839} 1840 1841/// ditto 1842auto byChunkAsync(Conn = AutoProtocol) 1843 (const(char)[] url, 1844 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1845 Conn conn = Conn()) 1846if (isCurlConn!(Conn)) 1847{ 1848 static if (is(Conn : AutoProtocol)) 1849 { 1850 if (isFTPUrl(url)) 1851 return byChunkAsync(url, cast(void[]) null, chunkSize, 1852 transmitBuffers, FTP()); 1853 else 1854 return byChunkAsync(url, cast(void[]) null, chunkSize, 1855 transmitBuffers, HTTP()); 1856 } 1857 else 1858 { 1859 return byChunkAsync(url, cast(void[]) null, chunkSize, 1860 transmitBuffers, conn); 1861 } 1862} 1863 1864@system unittest 1865{ 1866 import std.algorithm.comparison : equal; 1867 1868 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1869 { 1870 testServer.handle((s) { 1871 auto req = s.recvReq; 1872 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1873 }); 1874 assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1875 } 1876} 1877 1878 1879/* Used by byLineAsync/byChunkAsync to duplicate an existing connection 1880 * that can be used exclusively in a spawned thread. 1881 */ 1882private void _asyncDuplicateConnection(Conn, PostData) 1883 (const(char)[] url, Conn conn, PostData postData, Tid tid) 1884{ 1885 // no move semantic available in std.concurrency ie. must use casting. 1886 auto connDup = conn.dup(); 1887 connDup.url = url; 1888 1889 static if ( is(Conn : HTTP) ) 1890 { 1891 connDup.p.headersOut = null; 1892 connDup.method = conn.method == HTTP.Method.undefined ? 1893 HTTP.Method.get : conn.method; 1894 if (postData !is null) 1895 { 1896 if (connDup.method == HTTP.Method.put) 1897 { 1898 connDup.handle.set(CurlOption.infilesize_large, 1899 postData.length); 1900 } 1901 else 1902 { 1903 // post 1904 connDup.method = HTTP.Method.post; 1905 connDup.handle.set(CurlOption.postfieldsize_large, 1906 postData.length); 1907 } 1908 connDup.handle.set(CurlOption.copypostfields, 1909 cast(void*) postData.ptr); 1910 } 1911 tid.send(cast(ulong) connDup.handle.handle); 1912 tid.send(connDup.method); 1913 } 1914 else 1915 { 1916 enforce!CurlException(postData is null, 1917 "Cannot put ftp data using byLineAsync()"); 1918 tid.send(cast(ulong) connDup.handle.handle); 1919 tid.send(HTTP.Method.undefined); 1920 } 1921 connDup.p.curl.handle = null; // make sure handle is not freed 1922} 1923 1924/* 1925 Mixin template for all supported curl protocols. This is the commom 1926 functionallity such as timeouts and network interface settings. This should 1927 really be in the HTTP/FTP/SMTP structs but the documentation tool does not 1928 support a mixin to put its doc strings where a mixin is done. Therefore docs 1929 in this template is copied into each of HTTP/FTP/SMTP below. 1930*/ 1931private mixin template Protocol() 1932{ 1933 1934 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to 1935 /// pause a request 1936 alias requestPause = CurlReadFunc.pause; 1937 1938 /// Value to return from onSend delegate in order to abort a request 1939 alias requestAbort = CurlReadFunc.abort; 1940 1941 static uint defaultAsyncStringBufferSize = 100; 1942 1943 /** 1944 The curl handle used by this connection. 1945 */ 1946 @property ref Curl handle() return 1947 { 1948 return p.curl; 1949 } 1950 1951 /** 1952 True if the instance is stopped. A stopped instance is not usable. 1953 */ 1954 @property bool isStopped() 1955 { 1956 return p.curl.stopped; 1957 } 1958 1959 /// Stop and invalidate this instance. 1960 void shutdown() 1961 { 1962 p.curl.shutdown(); 1963 } 1964 1965 /** Set verbose. 1966 This will print request information to stderr. 1967 */ 1968 @property void verbose(bool on) 1969 { 1970 p.curl.set(CurlOption.verbose, on ? 1L : 0L); 1971 } 1972 1973 // Connection settings 1974 1975 /// Set timeout for activity on connection. 1976 @property void dataTimeout(Duration d) 1977 { 1978 p.curl.set(CurlOption.low_speed_limit, 1); 1979 p.curl.set(CurlOption.low_speed_time, d.total!"seconds"); 1980 } 1981 1982 /** Set maximum time an operation is allowed to take. 1983 This includes dns resolution, connecting, data transfer, etc. 1984 */ 1985 @property void operationTimeout(Duration d) 1986 { 1987 p.curl.set(CurlOption.timeout_ms, d.total!"msecs"); 1988 } 1989 1990 /// Set timeout for connecting. 1991 @property void connectTimeout(Duration d) 1992 { 1993 p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs"); 1994 } 1995 1996 // Network settings 1997 1998 /** Proxy 1999 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 2000 */ 2001 @property void proxy(const(char)[] host) 2002 { 2003 p.curl.set(CurlOption.proxy, host); 2004 } 2005 2006 /** Proxy port 2007 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 2008 */ 2009 @property void proxyPort(ushort port) 2010 { 2011 p.curl.set(CurlOption.proxyport, cast(long) port); 2012 } 2013 2014 /// Type of proxy 2015 alias CurlProxy = etc.c.curl.CurlProxy; 2016 2017 /** Proxy type 2018 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 2019 */ 2020 @property void proxyType(CurlProxy type) 2021 { 2022 p.curl.set(CurlOption.proxytype, cast(long) type); 2023 } 2024 2025 /// DNS lookup timeout. 2026 @property void dnsTimeout(Duration d) 2027 { 2028 p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs"); 2029 } 2030 2031 /** 2032 * The network interface to use in form of the the IP of the interface. 2033 * 2034 * Example: 2035 * ---- 2036 * theprotocol.netInterface = "192.168.1.32"; 2037 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 2038 * ---- 2039 * 2040 * See: $(REF InternetAddress, std,socket) 2041 */ 2042 @property void netInterface(const(char)[] i) 2043 { 2044 p.curl.set(CurlOption.intrface, i); 2045 } 2046 2047 /// ditto 2048 @property void netInterface(const(ubyte)[4] i) 2049 { 2050 import std.format : format; 2051 const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]); 2052 netInterface = str; 2053 } 2054 2055 /// ditto 2056 @property void netInterface(InternetAddress i) 2057 { 2058 netInterface = i.toAddrString(); 2059 } 2060 2061 /** 2062 Set the local outgoing port to use. 2063 Params: 2064 port = the first outgoing port number to try and use 2065 */ 2066 @property void localPort(ushort port) 2067 { 2068 p.curl.set(CurlOption.localport, cast(long) port); 2069 } 2070 2071 /** 2072 Set the no proxy flag for the specified host names. 2073 Params: 2074 test = a list of comma host names that do not require 2075 proxy to get reached 2076 */ 2077 void setNoProxy(string hosts) 2078 { 2079 p.curl.set(CurlOption.noproxy, hosts); 2080 } 2081 2082 /** 2083 Set the local outgoing port range to use. 2084 This can be used together with the localPort property. 2085 Params: 2086 range = if the first port is occupied then try this many 2087 port number forwards 2088 */ 2089 @property void localPortRange(ushort range) 2090 { 2091 p.curl.set(CurlOption.localportrange, cast(long) range); 2092 } 2093 2094 /** Set the tcp no-delay socket option on or off. 2095 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2096 */ 2097 @property void tcpNoDelay(bool on) 2098 { 2099 p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) ); 2100 } 2101 2102 /** Sets whether SSL peer certificates should be verified. 2103 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer) 2104 */ 2105 @property void verifyPeer(bool on) 2106 { 2107 p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0); 2108 } 2109 2110 /** Sets whether the host within an SSL certificate should be verified. 2111 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer) 2112 */ 2113 @property void verifyHost(bool on) 2114 { 2115 p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0); 2116 } 2117 2118 // Authentication settings 2119 2120 /** 2121 Set the user name, password and optionally domain for authentication 2122 purposes. 2123 2124 Some protocols may need authentication in some cases. Use this 2125 function to provide credentials. 2126 2127 Params: 2128 username = the username 2129 password = the password 2130 domain = used for NTLM authentication only and is set to the NTLM domain 2131 name 2132 */ 2133 void setAuthentication(const(char)[] username, const(char)[] password, 2134 const(char)[] domain = "") 2135 { 2136 import std.format : format; 2137 if (!domain.empty) 2138 username = format("%s/%s", domain, username); 2139 p.curl.set(CurlOption.userpwd, format("%s:%s", username, password)); 2140 } 2141 2142 @system unittest 2143 { 2144 import std.algorithm.searching : canFind; 2145 2146 testServer.handle((s) { 2147 auto req = s.recvReq; 2148 assert(req.hdrs.canFind("GET /")); 2149 assert(req.hdrs.canFind("Basic dXNlcjpwYXNz")); 2150 s.send(httpOK()); 2151 }); 2152 2153 auto http = HTTP(testServer.addr); 2154 http.onReceive = (ubyte[] data) { return data.length; }; 2155 http.setAuthentication("user", "pass"); 2156 http.perform(); 2157 2158 // Bugzilla 17540 2159 http.setNoProxy("www.example.com"); 2160 } 2161 2162 /** 2163 Set the user name and password for proxy authentication. 2164 2165 Params: 2166 username = the username 2167 password = the password 2168 */ 2169 void setProxyAuthentication(const(char)[] username, const(char)[] password) 2170 { 2171 import std.array : replace; 2172 import std.format : format; 2173 2174 p.curl.set(CurlOption.proxyuserpwd, 2175 format("%s:%s", 2176 username.replace(":", "%3A"), 2177 password.replace(":", "%3A")) 2178 ); 2179 } 2180 2181 /** 2182 * The event handler that gets called when data is needed for sending. The 2183 * length of the $(D void[]) specifies the maximum number of bytes that can 2184 * be sent. 2185 * 2186 * Returns: 2187 * The callback returns the number of elements in the buffer that have been 2188 * filled and are ready to send. 2189 * The special value $(D .abortRequest) can be returned in order to abort the 2190 * current request. 2191 * The special value $(D .pauseRequest) can be returned in order to pause the 2192 * current request. 2193 * 2194 * Example: 2195 * ---- 2196 * import std.net.curl; 2197 * string msg = "Hello world"; 2198 * auto client = HTTP("dlang.org"); 2199 * client.onSend = delegate size_t(void[] data) 2200 * { 2201 * auto m = cast(void[]) msg; 2202 * size_t length = m.length > data.length ? data.length : m.length; 2203 * if (length == 0) return 0; 2204 * data[0 .. length] = m[0 .. length]; 2205 * msg = msg[length..$]; 2206 * return length; 2207 * }; 2208 * client.perform(); 2209 * ---- 2210 */ 2211 @property void onSend(size_t delegate(void[]) callback) 2212 { 2213 p.curl.clear(CurlOption.postfields); // cannot specify data when using callback 2214 p.curl.onSend = callback; 2215 } 2216 2217 /** 2218 * The event handler that receives incoming data. Be sure to copy the 2219 * incoming ubyte[] since it is not guaranteed to be valid after the 2220 * callback returns. 2221 * 2222 * Returns: 2223 * The callback returns the number of incoming bytes read. If the entire array is 2224 * not read the request will abort. 2225 * The special value .pauseRequest can be returned in order to pause the 2226 * current request. 2227 * 2228 * Example: 2229 * ---- 2230 * import std.net.curl, std.stdio; 2231 * auto client = HTTP("dlang.org"); 2232 * client.onReceive = (ubyte[] data) 2233 * { 2234 * writeln("Got data", to!(const(char)[])(data)); 2235 * return data.length; 2236 * }; 2237 * client.perform(); 2238 * ---- 2239 */ 2240 @property void onReceive(size_t delegate(ubyte[]) callback) 2241 { 2242 p.curl.onReceive = callback; 2243 } 2244 2245 /** 2246 * The event handler that gets called to inform of upload/download progress. 2247 * 2248 * Params: 2249 * dlTotal = total bytes to download 2250 * dlNow = currently downloaded bytes 2251 * ulTotal = total bytes to upload 2252 * ulNow = currently uploaded bytes 2253 * 2254 * Returns: 2255 * Return 0 from the callback to signal success, return non-zero to abort 2256 * transfer 2257 * 2258 * Example: 2259 * ---- 2260 * import std.net.curl, std.stdio; 2261 * auto client = HTTP("dlang.org"); 2262 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult) 2263 * { 2264 * writeln("Progress: downloaded ", dln, " of ", dl); 2265 * writeln("Progress: uploaded ", uln, " of ", ul); 2266 * }; 2267 * client.perform(); 2268 * ---- 2269 */ 2270 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2271 size_t ulTotal, size_t ulNow) callback) 2272 { 2273 p.curl.onProgress = callback; 2274 } 2275} 2276 2277/* 2278 Decode $(D ubyte[]) array using the provided EncodingScheme up to maxChars 2279 Returns: Tuple of ubytes read and the $(D Char[]) characters decoded. 2280 Not all ubytes are guaranteed to be read in case of decoding error. 2281*/ 2282private Tuple!(size_t,Char[]) 2283decodeString(Char = char)(const(ubyte)[] data, 2284 EncodingScheme scheme, 2285 size_t maxChars = size_t.max) 2286{ 2287 Char[] res; 2288 immutable startLen = data.length; 2289 size_t charsDecoded = 0; 2290 while (data.length && charsDecoded < maxChars) 2291 { 2292 immutable dchar dc = scheme.safeDecode(data); 2293 if (dc == INVALID_SEQUENCE) 2294 { 2295 return typeof(return)(size_t.max, cast(Char[]) null); 2296 } 2297 charsDecoded++; 2298 res ~= dc; 2299 } 2300 return typeof(return)(startLen-data.length, res); 2301} 2302 2303/* 2304 Decode $(D ubyte[]) array using the provided $(D EncodingScheme) until a the 2305 line terminator specified is found. The basesrc parameter is effectively 2306 prepended to src as the first thing. 2307 2308 This function is used for decoding as much of the src buffer as 2309 possible until either the terminator is found or decoding fails. If 2310 it fails as the last data in the src it may mean that the src buffer 2311 were missing some bytes in order to represent a correct code 2312 point. Upon the next call to this function more bytes have been 2313 received from net and the failing bytes should be given as the 2314 basesrc parameter. It is done this way to minimize data copying. 2315 2316 Returns: true if a terminator was found 2317 Not all ubytes are guaranteed to be read in case of decoding error. 2318 any decoded chars will be inserted into dst. 2319*/ 2320private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc, 2321 ref const(ubyte)[] src, 2322 ref Char[] dst, 2323 EncodingScheme scheme, 2324 Terminator terminator) 2325{ 2326 import std.algorithm.searching : endsWith; 2327 2328 // if there is anything in the basesrc then try to decode that 2329 // first. 2330 if (basesrc.length != 0) 2331 { 2332 // Try to ensure 4 entries in the basesrc by copying from src. 2333 immutable blen = basesrc.length; 2334 immutable len = (basesrc.length + src.length) >= 4 ? 2335 4 : basesrc.length + src.length; 2336 basesrc.length = len; 2337 2338 immutable dchar dc = scheme.safeDecode(basesrc); 2339 if (dc == INVALID_SEQUENCE) 2340 { 2341 enforce!CurlException(len != 4, "Invalid code sequence"); 2342 return false; 2343 } 2344 dst ~= dc; 2345 src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src 2346 basesrc.length = 0; 2347 } 2348 2349 while (src.length) 2350 { 2351 const lsrc = src; 2352 dchar dc = scheme.safeDecode(src); 2353 if (dc == INVALID_SEQUENCE) 2354 { 2355 if (src.empty) 2356 { 2357 // The invalid sequence was in the end of the src. Maybe there 2358 // just need to be more bytes available so these last bytes are 2359 // put back to src for later use. 2360 src = lsrc; 2361 return false; 2362 } 2363 dc = '?'; 2364 } 2365 dst ~= dc; 2366 2367 if (dst.endsWith(terminator)) 2368 return true; 2369 } 2370 return false; // no terminator found 2371} 2372 2373/** 2374 * HTTP client functionality. 2375 * 2376 * Example: 2377 * --- 2378 * import std.net.curl, std.stdio; 2379 * 2380 * // Get with custom data receivers 2381 * auto http = HTTP("dlang.org"); 2382 * http.onReceiveHeader = 2383 * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); }; 2384 * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 2385 * http.perform(); 2386 * 2387 * // Put with data senders 2388 * auto msg = "Hello world"; 2389 * http.contentLength = msg.length; 2390 * http.onSend = (void[] data) 2391 * { 2392 * auto m = cast(void[]) msg; 2393 * size_t len = m.length > data.length ? data.length : m.length; 2394 * if (len == 0) return len; 2395 * data[0 .. len] = m[0 .. len]; 2396 * msg = msg[len..$]; 2397 * return len; 2398 * }; 2399 * http.perform(); 2400 * 2401 * // Track progress 2402 * http.method = HTTP.Method.get; 2403 * http.url = "http://upload.wikimedia.org/wikipedia/commons/" 2404 * "5/53/Wikipedia-logo-en-big.png"; 2405 * http.onReceive = (ubyte[] data) { return data.length; }; 2406 * http.onProgress = (size_t dltotal, size_t dlnow, 2407 * size_t ultotal, size_t ulnow) 2408 * { 2409 * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); 2410 * return 0; 2411 * }; 2412 * http.perform(); 2413 * --- 2414 * 2415 * See_Also: $(_HTTP www.ietf.org/rfc/rfc2616.txt, RFC2616) 2416 * 2417 */ 2418struct HTTP 2419{ 2420 mixin Protocol; 2421 2422 import std.datetime.systime : SysTime; 2423 2424 /// Authentication method equal to $(REF CurlAuth, etc,c,curl) 2425 alias AuthMethod = CurlAuth; 2426 2427 static private uint defaultMaxRedirects = 10; 2428 2429 private struct Impl 2430 { 2431 ~this() 2432 { 2433 if (headersOut !is null) 2434 Curl.curl.slist_free_all(headersOut); 2435 if (curl.handle !is null) // work around RefCounted/emplace bug 2436 curl.shutdown(); 2437 } 2438 Curl curl; 2439 curl_slist* headersOut; 2440 string[string] headersIn; 2441 string charset; 2442 2443 /// The status line of the final sub-request in a request. 2444 StatusLine status; 2445 private void delegate(StatusLine) onReceiveStatusLine; 2446 2447 /// The HTTP method to use. 2448 Method method = Method.undefined; 2449 2450 @system @property void onReceiveHeader(void delegate(in char[] key, 2451 in char[] value) callback) 2452 { 2453 import std.algorithm.searching : startsWith; 2454 import std.regex : regex, match; 2455 import std.uni : toLower; 2456 2457 // Wrap incoming callback in order to separate http status line from 2458 // http headers. On redirected requests there may be several such 2459 // status lines. The last one is the one recorded. 2460 auto dg = (in char[] header) 2461 { 2462 import std.utf : UTFException; 2463 try 2464 { 2465 if (header.empty) 2466 { 2467 // header delimiter 2468 return; 2469 } 2470 if (header.startsWith("HTTP/")) 2471 { 2472 headersIn.clear(); 2473 if (parseStatusLine(header, status)) 2474 { 2475 if (onReceiveStatusLine != null) 2476 onReceiveStatusLine(status); 2477 } 2478 return; 2479 } 2480 2481 // Normal http header 2482 auto m = match(cast(char[]) header, regex("(.*?): (.*)$")); 2483 2484 auto fieldName = m.captures[1].toLower().idup; 2485 if (fieldName == "content-type") 2486 { 2487 auto mct = match(cast(char[]) m.captures[2], 2488 regex("charset=([^;]*)", "i")); 2489 if (!mct.empty && mct.captures.length > 1) 2490 charset = mct.captures[1].idup; 2491 } 2492 2493 if (!m.empty && callback !is null) 2494 callback(fieldName, m.captures[2]); 2495 headersIn[fieldName] = m.captures[2].idup; 2496 } 2497 catch (UTFException e) 2498 { 2499 //munch it - a header should be all ASCII, any "wrong UTF" is broken header 2500 } 2501 }; 2502 2503 curl.onReceiveHeader = dg; 2504 } 2505 } 2506 2507 private RefCounted!Impl p; 2508 2509 /// Parse status line, as received from / generated by cURL. 2510 private static bool parseStatusLine(in char[] header, out StatusLine status) @safe 2511 { 2512 import std.conv : to; 2513 import std.regex : regex, match; 2514 2515 const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$")); 2516 if (m.empty) 2517 return false; // Invalid status line 2518 else 2519 { 2520 status.majorVersion = to!ushort(m.captures[1]); 2521 status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0; 2522 status.code = to!ushort(m.captures[3]); 2523 status.reason = m.captures[4].idup; 2524 return true; 2525 } 2526 } 2527 2528 @safe unittest 2529 { 2530 StatusLine status; 2531 assert(parseStatusLine("HTTP/1.1 200 OK", status) 2532 && status == StatusLine(1, 1, 200, "OK")); 2533 assert(parseStatusLine("HTTP/1.0 304 Not Modified", status) 2534 && status == StatusLine(1, 0, 304, "Not Modified")); 2535 // The HTTP2 protocol is binary; cURL generates this fake text header. 2536 assert(parseStatusLine("HTTP/2 200", status) 2537 && status == StatusLine(2, 0, 200, null)); 2538 } 2539 2540 /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl) 2541 2542 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 2543 */ 2544 alias TimeCond = CurlTimeCond; 2545 2546 /** 2547 Constructor taking the url as parameter. 2548 */ 2549 static HTTP opCall(const(char)[] url) 2550 { 2551 HTTP http; 2552 http.initialize(); 2553 http.url = url; 2554 return http; 2555 } 2556 2557 /// 2558 static HTTP opCall() 2559 { 2560 HTTP http; 2561 http.initialize(); 2562 return http; 2563 } 2564 2565 /// 2566 HTTP dup() 2567 { 2568 HTTP copy; 2569 copy.initialize(); 2570 copy.p.method = p.method; 2571 curl_slist* cur = p.headersOut; 2572 curl_slist* newlist = null; 2573 while (cur) 2574 { 2575 newlist = Curl.curl.slist_append(newlist, cur.data); 2576 cur = cur.next; 2577 } 2578 copy.p.headersOut = newlist; 2579 copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut); 2580 copy.p.curl = p.curl.dup(); 2581 copy.dataTimeout = _defaultDataTimeout; 2582 copy.onReceiveHeader = null; 2583 return copy; 2584 } 2585 2586 private void initialize() 2587 { 2588 p.curl.initialize(); 2589 maxRedirects = HTTP.defaultMaxRedirects; 2590 p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC 2591 p.method = Method.undefined; 2592 setUserAgent(HTTP.defaultUserAgent); 2593 dataTimeout = _defaultDataTimeout; 2594 onReceiveHeader = null; 2595 verifyPeer = true; 2596 verifyHost = true; 2597 } 2598 2599 /** 2600 Perform a http request. 2601 2602 After the HTTP client has been setup and possibly assigned callbacks the 2603 $(D perform()) method will start performing the request towards the 2604 specified server. 2605 2606 Params: 2607 throwOnError = whether to throw an exception or return a CurlCode on error 2608 */ 2609 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 2610 { 2611 p.status.reset(); 2612 2613 CurlOption opt; 2614 final switch (p.method) 2615 { 2616 case Method.head: 2617 p.curl.set(CurlOption.nobody, 1L); 2618 opt = CurlOption.nobody; 2619 break; 2620 case Method.undefined: 2621 case Method.get: 2622 p.curl.set(CurlOption.httpget, 1L); 2623 opt = CurlOption.httpget; 2624 break; 2625 case Method.post: 2626 p.curl.set(CurlOption.post, 1L); 2627 opt = CurlOption.post; 2628 break; 2629 case Method.put: 2630 p.curl.set(CurlOption.upload, 1L); 2631 opt = CurlOption.upload; 2632 break; 2633 case Method.del: 2634 p.curl.set(CurlOption.customrequest, "DELETE"); 2635 opt = CurlOption.customrequest; 2636 break; 2637 case Method.options: 2638 p.curl.set(CurlOption.customrequest, "OPTIONS"); 2639 opt = CurlOption.customrequest; 2640 break; 2641 case Method.trace: 2642 p.curl.set(CurlOption.customrequest, "TRACE"); 2643 opt = CurlOption.customrequest; 2644 break; 2645 case Method.connect: 2646 p.curl.set(CurlOption.customrequest, "CONNECT"); 2647 opt = CurlOption.customrequest; 2648 break; 2649 case Method.patch: 2650 p.curl.set(CurlOption.customrequest, "PATCH"); 2651 opt = CurlOption.customrequest; 2652 break; 2653 } 2654 2655 scope (exit) p.curl.clear(opt); 2656 return p.curl.perform(throwOnError); 2657 } 2658 2659 /// The URL to specify the location of the resource. 2660 @property void url(const(char)[] url) 2661 { 2662 import std.algorithm.searching : startsWith; 2663 import std.uni : toLower; 2664 if (!startsWith(url.toLower(), "http://", "https://")) 2665 url = "http://" ~ url; 2666 p.curl.set(CurlOption.url, url); 2667 } 2668 2669 /// Set the CA certificate bundle file to use for SSL peer verification 2670 @property void caInfo(const(char)[] caFile) 2671 { 2672 p.curl.set(CurlOption.cainfo, caFile); 2673 } 2674 2675 // This is a workaround for mixed in content not having its 2676 // docs mixed in. 2677 version (StdDdoc) 2678 { 2679 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to 2680 /// pause a request 2681 alias requestPause = CurlReadFunc.pause; 2682 2683 /// Value to return from onSend delegate in order to abort a request 2684 alias requestAbort = CurlReadFunc.abort; 2685 2686 /** 2687 True if the instance is stopped. A stopped instance is not usable. 2688 */ 2689 @property bool isStopped(); 2690 2691 /// Stop and invalidate this instance. 2692 void shutdown(); 2693 2694 /** Set verbose. 2695 This will print request information to stderr. 2696 */ 2697 @property void verbose(bool on); 2698 2699 // Connection settings 2700 2701 /// Set timeout for activity on connection. 2702 @property void dataTimeout(Duration d); 2703 2704 /** Set maximum time an operation is allowed to take. 2705 This includes dns resolution, connecting, data transfer, etc. 2706 */ 2707 @property void operationTimeout(Duration d); 2708 2709 /// Set timeout for connecting. 2710 @property void connectTimeout(Duration d); 2711 2712 // Network settings 2713 2714 /** Proxy 2715 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 2716 */ 2717 @property void proxy(const(char)[] host); 2718 2719 /** Proxy port 2720 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 2721 */ 2722 @property void proxyPort(ushort port); 2723 2724 /// Type of proxy 2725 alias CurlProxy = etc.c.curl.CurlProxy; 2726 2727 /** Proxy type 2728 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 2729 */ 2730 @property void proxyType(CurlProxy type); 2731 2732 /// DNS lookup timeout. 2733 @property void dnsTimeout(Duration d); 2734 2735 /** 2736 * The network interface to use in form of the the IP of the interface. 2737 * 2738 * Example: 2739 * ---- 2740 * theprotocol.netInterface = "192.168.1.32"; 2741 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 2742 * ---- 2743 * 2744 * See: $(REF InternetAddress, std,socket) 2745 */ 2746 @property void netInterface(const(char)[] i); 2747 2748 /// ditto 2749 @property void netInterface(const(ubyte)[4] i); 2750 2751 /// ditto 2752 @property void netInterface(InternetAddress i); 2753 2754 /** 2755 Set the local outgoing port to use. 2756 Params: 2757 port = the first outgoing port number to try and use 2758 */ 2759 @property void localPort(ushort port); 2760 2761 /** 2762 Set the local outgoing port range to use. 2763 This can be used together with the localPort property. 2764 Params: 2765 range = if the first port is occupied then try this many 2766 port number forwards 2767 */ 2768 @property void localPortRange(ushort range); 2769 2770 /** Set the tcp no-delay socket option on or off. 2771 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2772 */ 2773 @property void tcpNoDelay(bool on); 2774 2775 // Authentication settings 2776 2777 /** 2778 Set the user name, password and optionally domain for authentication 2779 purposes. 2780 2781 Some protocols may need authentication in some cases. Use this 2782 function to provide credentials. 2783 2784 Params: 2785 username = the username 2786 password = the password 2787 domain = used for NTLM authentication only and is set to the NTLM domain 2788 name 2789 */ 2790 void setAuthentication(const(char)[] username, const(char)[] password, 2791 const(char)[] domain = ""); 2792 2793 /** 2794 Set the user name and password for proxy authentication. 2795 2796 Params: 2797 username = the username 2798 password = the password 2799 */ 2800 void setProxyAuthentication(const(char)[] username, const(char)[] password); 2801 2802 /** 2803 * The event handler that gets called when data is needed for sending. The 2804 * length of the $(D void[]) specifies the maximum number of bytes that can 2805 * be sent. 2806 * 2807 * Returns: 2808 * The callback returns the number of elements in the buffer that have been 2809 * filled and are ready to send. 2810 * The special value $(D .abortRequest) can be returned in order to abort the 2811 * current request. 2812 * The special value $(D .pauseRequest) can be returned in order to pause the 2813 * current request. 2814 * 2815 * Example: 2816 * ---- 2817 * import std.net.curl; 2818 * string msg = "Hello world"; 2819 * auto client = HTTP("dlang.org"); 2820 * client.onSend = delegate size_t(void[] data) 2821 * { 2822 * auto m = cast(void[]) msg; 2823 * size_t length = m.length > data.length ? data.length : m.length; 2824 * if (length == 0) return 0; 2825 * data[0 .. length] = m[0 .. length]; 2826 * msg = msg[length..$]; 2827 * return length; 2828 * }; 2829 * client.perform(); 2830 * ---- 2831 */ 2832 @property void onSend(size_t delegate(void[]) callback); 2833 2834 /** 2835 * The event handler that receives incoming data. Be sure to copy the 2836 * incoming ubyte[] since it is not guaranteed to be valid after the 2837 * callback returns. 2838 * 2839 * Returns: 2840 * The callback returns the incoming bytes read. If not the entire array is 2841 * the request will abort. 2842 * The special value .pauseRequest can be returned in order to pause the 2843 * current request. 2844 * 2845 * Example: 2846 * ---- 2847 * import std.net.curl, std.stdio; 2848 * auto client = HTTP("dlang.org"); 2849 * client.onReceive = (ubyte[] data) 2850 * { 2851 * writeln("Got data", to!(const(char)[])(data)); 2852 * return data.length; 2853 * }; 2854 * client.perform(); 2855 * ---- 2856 */ 2857 @property void onReceive(size_t delegate(ubyte[]) callback); 2858 2859 /** 2860 * Register an event handler that gets called to inform of 2861 * upload/download progress. 2862 * 2863 * Callback_parameters: 2864 * $(CALLBACK_PARAMS) 2865 * 2866 * Callback_returns: Return 0 to signal success, return non-zero to 2867 * abort transfer. 2868 * 2869 * Example: 2870 * ---- 2871 * import std.net.curl, std.stdio; 2872 * auto client = HTTP("dlang.org"); 2873 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult) 2874 * { 2875 * writeln("Progress: downloaded ", dln, " of ", dl); 2876 * writeln("Progress: uploaded ", uln, " of ", ul); 2877 * }; 2878 * client.perform(); 2879 * ---- 2880 */ 2881 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2882 size_t ulTotal, size_t ulNow) callback); 2883 } 2884 2885 /** Clear all outgoing headers. 2886 */ 2887 void clearRequestHeaders() 2888 { 2889 if (p.headersOut !is null) 2890 Curl.curl.slist_free_all(p.headersOut); 2891 p.headersOut = null; 2892 p.curl.clear(CurlOption.httpheader); 2893 } 2894 2895 /** Add a header e.g. "X-CustomField: Something is fishy". 2896 * 2897 * There is no remove header functionality. Do a $(LREF clearRequestHeaders) 2898 * and set the needed headers instead. 2899 * 2900 * Example: 2901 * --- 2902 * import std.net.curl; 2903 * auto client = HTTP(); 2904 * client.addRequestHeader("X-Custom-ABC", "This is the custom value"); 2905 * auto content = get("dlang.org", client); 2906 * --- 2907 */ 2908 void addRequestHeader(const(char)[] name, const(char)[] value) 2909 { 2910 import std.format : format; 2911 import std.uni : icmp; 2912 2913 if (icmp(name, "User-Agent") == 0) 2914 return setUserAgent(value); 2915 string nv = format("%s: %s", name, value); 2916 p.headersOut = Curl.curl.slist_append(p.headersOut, 2917 nv.tempCString().buffPtr); 2918 p.curl.set(CurlOption.httpheader, p.headersOut); 2919 } 2920 2921 /** 2922 * The default "User-Agent" value send with a request. 2923 * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))" 2924 */ 2925 static string defaultUserAgent() @property 2926 { 2927 import std.compiler : version_major, version_minor; 2928 import std.format : format, sformat; 2929 2930 // http://curl.haxx.se/docs/versions.html 2931 enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)"; 2932 enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3; 2933 2934 static char[maxLen] buf = void; 2935 static string userAgent; 2936 2937 if (!userAgent.length) 2938 { 2939 auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num; 2940 userAgent = cast(immutable) sformat( 2941 buf, fmt, version_major, version_minor, 2942 curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF); 2943 } 2944 return userAgent; 2945 } 2946 2947 /** Set the value of the user agent request header field. 2948 * 2949 * By default a request has it's "User-Agent" field set to $(LREF 2950 * defaultUserAgent) even if $(D setUserAgent) was never called. Pass 2951 * an empty string to suppress the "User-Agent" field altogether. 2952 */ 2953 void setUserAgent(const(char)[] userAgent) 2954 { 2955 p.curl.set(CurlOption.useragent, userAgent); 2956 } 2957 2958 /** 2959 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 2960 * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok). 2961 * 2962 * Params: 2963 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 2964 * The values are: 2965 * $(D etc.c.curl.CurlInfo.namelookup_time), 2966 * $(D etc.c.curl.CurlInfo.connect_time), 2967 * $(D etc.c.curl.CurlInfo.pretransfer_time), 2968 * $(D etc.c.curl.CurlInfo.starttransfer_time), 2969 * $(D etc.c.curl.CurlInfo.redirect_time), 2970 * $(D etc.c.curl.CurlInfo.appconnect_time), 2971 * $(D etc.c.curl.CurlInfo.total_time). 2972 * val = the actual value of the inquired timing. 2973 * 2974 * Returns: 2975 * The return code of the operation. The value stored in val 2976 * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok). 2977 * 2978 * Example: 2979 * --- 2980 * import std.net.curl; 2981 * import etc.c.curl : CurlError, CurlInfo; 2982 * 2983 * auto client = HTTP("dlang.org"); 2984 * client.perform(); 2985 * 2986 * double val; 2987 * CurlCode code; 2988 * 2989 * code = http.getTiming(CurlInfo.namelookup_time, val); 2990 * assert(code == CurlError.ok); 2991 * --- 2992 */ 2993 CurlCode getTiming(CurlInfo timing, ref double val) 2994 { 2995 return p.curl.getTiming(timing, val); 2996 } 2997 2998 /** The headers read from a successful response. 2999 * 3000 */ 3001 @property string[string] responseHeaders() 3002 { 3003 return p.headersIn; 3004 } 3005 3006 /// HTTP method used. 3007 @property void method(Method m) 3008 { 3009 p.method = m; 3010 } 3011 3012 /// ditto 3013 @property Method method() 3014 { 3015 return p.method; 3016 } 3017 3018 /** 3019 HTTP status line of last response. One call to perform may 3020 result in several requests because of redirection. 3021 */ 3022 @property StatusLine statusLine() 3023 { 3024 return p.status; 3025 } 3026 3027 /// Set the active cookie string e.g. "name1=value1;name2=value2" 3028 void setCookie(const(char)[] cookie) 3029 { 3030 p.curl.set(CurlOption.cookie, cookie); 3031 } 3032 3033 /// Set a file path to where a cookie jar should be read/stored. 3034 void setCookieJar(const(char)[] path) 3035 { 3036 p.curl.set(CurlOption.cookiefile, path); 3037 if (path.length) 3038 p.curl.set(CurlOption.cookiejar, path); 3039 } 3040 3041 /// Flush cookie jar to disk. 3042 void flushCookieJar() 3043 { 3044 p.curl.set(CurlOption.cookielist, "FLUSH"); 3045 } 3046 3047 /// Clear session cookies. 3048 void clearSessionCookies() 3049 { 3050 p.curl.set(CurlOption.cookielist, "SESS"); 3051 } 3052 3053 /// Clear all cookies. 3054 void clearAllCookies() 3055 { 3056 p.curl.set(CurlOption.cookielist, "ALL"); 3057 } 3058 3059 /** 3060 Set time condition on the request. 3061 3062 Params: 3063 cond = $(D CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}) 3064 timestamp = Timestamp for the condition 3065 3066 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 3067 */ 3068 void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp) 3069 { 3070 p.curl.set(CurlOption.timecondition, cond); 3071 p.curl.set(CurlOption.timevalue, timestamp.toUnixTime()); 3072 } 3073 3074 /** Specifying data to post when not using the onSend callback. 3075 * 3076 * The data is NOT copied by the library. Content-Type will default to 3077 * application/octet-stream. Data is not converted or encoded by this 3078 * method. 3079 * 3080 * Example: 3081 * ---- 3082 * import std.net.curl, std.stdio; 3083 * auto http = HTTP("http://www.mydomain.com"); 3084 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3085 * http.postData = [1,2,3,4,5]; 3086 * http.perform(); 3087 * ---- 3088 */ 3089 @property void postData(const(void)[] data) 3090 { 3091 setPostData(data, "application/octet-stream"); 3092 } 3093 3094 /** Specifying data to post when not using the onSend callback. 3095 * 3096 * The data is NOT copied by the library. Content-Type will default to 3097 * text/plain. Data is not converted or encoded by this method. 3098 * 3099 * Example: 3100 * ---- 3101 * import std.net.curl, std.stdio; 3102 * auto http = HTTP("http://www.mydomain.com"); 3103 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3104 * http.postData = "The quick...."; 3105 * http.perform(); 3106 * ---- 3107 */ 3108 @property void postData(const(char)[] data) 3109 { 3110 setPostData(data, "text/plain"); 3111 } 3112 3113 /** 3114 * Specify data to post when not using the onSend callback, with 3115 * user-specified Content-Type. 3116 * Params: 3117 * data = Data to post. 3118 * contentType = MIME type of the data, for example, "text/plain" or 3119 * "application/octet-stream". See also: 3120 * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type, 3121 * Internet media type) on Wikipedia. 3122 * ----- 3123 * import std.net.curl; 3124 * auto http = HTTP("http://onlineform.example.com"); 3125 * auto data = "app=login&username=bob&password=s00perS3kret"; 3126 * http.setPostData(data, "application/x-www-form-urlencoded"); 3127 * http.onReceive = (ubyte[] data) { return data.length; }; 3128 * http.perform(); 3129 * ----- 3130 */ 3131 void setPostData(const(void)[] data, string contentType) 3132 { 3133 // cannot use callback when specifying data directly so it is disabled here. 3134 p.curl.clear(CurlOption.readfunction); 3135 addRequestHeader("Content-Type", contentType); 3136 p.curl.set(CurlOption.postfields, cast(void*) data.ptr); 3137 p.curl.set(CurlOption.postfieldsize, data.length); 3138 if (method == Method.undefined) 3139 method = Method.post; 3140 } 3141 3142 @system unittest 3143 { 3144 import std.algorithm.searching : canFind; 3145 3146 testServer.handle((s) { 3147 auto req = s.recvReq!ubyte; 3148 assert(req.hdrs.canFind("POST /path")); 3149 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 3150 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 3151 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 3152 }); 3153 auto data = new ubyte[](256); 3154 foreach (i, ref ub; data) 3155 ub = cast(ubyte) i; 3156 3157 auto http = HTTP(testServer.addr~"/path"); 3158 http.postData = data; 3159 ubyte[] res; 3160 http.onReceive = (data) { res ~= data; return data.length; }; 3161 http.perform(); 3162 assert(res == cast(ubyte[])[17, 27, 35, 41]); 3163 } 3164 3165 /** 3166 * Set the event handler that receives incoming headers. 3167 * 3168 * The callback will receive a header field key, value as parameter. The 3169 * $(D const(char)[]) arrays are not valid after the delegate has returned. 3170 * 3171 * Example: 3172 * ---- 3173 * import std.net.curl, std.stdio; 3174 * auto http = HTTP("dlang.org"); 3175 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3176 * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); }; 3177 * http.perform(); 3178 * ---- 3179 */ 3180 @property void onReceiveHeader(void delegate(in char[] key, 3181 in char[] value) callback) 3182 { 3183 p.onReceiveHeader = callback; 3184 } 3185 3186 /** 3187 Callback for each received StatusLine. 3188 3189 Notice that several callbacks can be done for each call to 3190 $(D perform()) due to redirections. 3191 3192 See_Also: $(LREF StatusLine) 3193 */ 3194 @property void onReceiveStatusLine(void delegate(StatusLine) callback) 3195 { 3196 p.onReceiveStatusLine = callback; 3197 } 3198 3199 /** 3200 The content length in bytes when using request that has content 3201 e.g. POST/PUT and not using chunked transfer. Is set as the 3202 "Content-Length" header. Set to ulong.max to reset to chunked transfer. 3203 */ 3204 @property void contentLength(ulong len) 3205 { 3206 import std.conv : to; 3207 3208 CurlOption lenOpt; 3209 3210 // Force post if necessary 3211 if (p.method != Method.put && p.method != Method.post && 3212 p.method != Method.patch) 3213 p.method = Method.post; 3214 3215 if (p.method == Method.post || p.method == Method.patch) 3216 lenOpt = CurlOption.postfieldsize_large; 3217 else 3218 lenOpt = CurlOption.infilesize_large; 3219 3220 if (size_t.max != ulong.max && len == size_t.max) 3221 len = ulong.max; // check size_t.max for backwards compat, turn into error 3222 3223 if (len == ulong.max) 3224 { 3225 // HTTP 1.1 supports requests with no length header set. 3226 addRequestHeader("Transfer-Encoding", "chunked"); 3227 addRequestHeader("Expect", "100-continue"); 3228 } 3229 else 3230 { 3231 p.curl.set(lenOpt, to!curl_off_t(len)); 3232 } 3233 } 3234 3235 /** 3236 Authentication method as specified in $(LREF AuthMethod). 3237 */ 3238 @property void authenticationMethod(AuthMethod authMethod) 3239 { 3240 p.curl.set(CurlOption.httpauth, cast(long) authMethod); 3241 } 3242 3243 /** 3244 Set max allowed redirections using the location header. 3245 uint.max for infinite. 3246 */ 3247 @property void maxRedirects(uint maxRedirs) 3248 { 3249 if (maxRedirs == uint.max) 3250 { 3251 // Disable 3252 p.curl.set(CurlOption.followlocation, 0); 3253 } 3254 else 3255 { 3256 p.curl.set(CurlOption.followlocation, 1); 3257 p.curl.set(CurlOption.maxredirs, maxRedirs); 3258 } 3259 } 3260 3261 /** <a name="HTTP.Method"/>The standard HTTP methods : 3262 * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1) 3263 */ 3264 enum Method 3265 { 3266 undefined, 3267 head, /// 3268 get, /// 3269 post, /// 3270 put, /// 3271 del, /// 3272 options, /// 3273 trace, /// 3274 connect, /// 3275 patch, /// 3276 } 3277 3278 /** 3279 HTTP status line ie. the first line returned in an HTTP response. 3280 3281 If authentication or redirections are done then the status will be for 3282 the last response received. 3283 */ 3284 struct StatusLine 3285 { 3286 ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0. 3287 ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0. 3288 ushort code; /// HTTP status line code e.g. 200. 3289 string reason; /// HTTP status line reason string. 3290 3291 /// Reset this status line 3292 @safe void reset() 3293 { 3294 majorVersion = 0; 3295 minorVersion = 0; 3296 code = 0; 3297 reason = ""; 3298 } 3299 3300 /// 3301 string toString() const 3302 { 3303 import std.format : format; 3304 return format("%s %s (%s.%s)", 3305 code, reason, majorVersion, minorVersion); 3306 } 3307 } 3308 3309} // HTTP 3310 3311@system unittest // charset/Charset/CHARSET/... 3312{ 3313 import std.meta : AliasSeq; 3314 3315 foreach (c; AliasSeq!("charset", "Charset", "CHARSET", "CharSet", "charSet", 3316 "ChArSeT", "cHaRsEt")) 3317 { 3318 testServer.handle((s) { 3319 s.send("HTTP/1.1 200 OK\r\n"~ 3320 "Content-Length: 0\r\n"~ 3321 "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~ 3322 "\r\n"); 3323 }); 3324 3325 auto http = HTTP(testServer.addr); 3326 http.perform(); 3327 assert(http.p.charset == "foo"); 3328 3329 // Bugzilla 16736 3330 double val; 3331 CurlCode code; 3332 3333 code = http.getTiming(CurlInfo.total_time, val); 3334 assert(code == CurlError.ok); 3335 code = http.getTiming(CurlInfo.namelookup_time, val); 3336 assert(code == CurlError.ok); 3337 code = http.getTiming(CurlInfo.connect_time, val); 3338 assert(code == CurlError.ok); 3339 code = http.getTiming(CurlInfo.pretransfer_time, val); 3340 assert(code == CurlError.ok); 3341 code = http.getTiming(CurlInfo.starttransfer_time, val); 3342 assert(code == CurlError.ok); 3343 code = http.getTiming(CurlInfo.redirect_time, val); 3344 assert(code == CurlError.ok); 3345 code = http.getTiming(CurlInfo.appconnect_time, val); 3346 assert(code == CurlError.ok); 3347 } 3348} 3349 3350/** 3351 FTP client functionality. 3352 3353 See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959) 3354*/ 3355struct FTP 3356{ 3357 3358 mixin Protocol; 3359 3360 private struct Impl 3361 { 3362 ~this() 3363 { 3364 if (commands !is null) 3365 Curl.curl.slist_free_all(commands); 3366 if (curl.handle !is null) // work around RefCounted/emplace bug 3367 curl.shutdown(); 3368 } 3369 curl_slist* commands; 3370 Curl curl; 3371 string encoding; 3372 } 3373 3374 private RefCounted!Impl p; 3375 3376 /** 3377 FTP access to the specified url. 3378 */ 3379 static FTP opCall(const(char)[] url) 3380 { 3381 FTP ftp; 3382 ftp.initialize(); 3383 ftp.url = url; 3384 return ftp; 3385 } 3386 3387 /// 3388 static FTP opCall() 3389 { 3390 FTP ftp; 3391 ftp.initialize(); 3392 return ftp; 3393 } 3394 3395 /// 3396 FTP dup() 3397 { 3398 FTP copy = FTP(); 3399 copy.initialize(); 3400 copy.p.encoding = p.encoding; 3401 copy.p.curl = p.curl.dup(); 3402 curl_slist* cur = p.commands; 3403 curl_slist* newlist = null; 3404 while (cur) 3405 { 3406 newlist = Curl.curl.slist_append(newlist, cur.data); 3407 cur = cur.next; 3408 } 3409 copy.p.commands = newlist; 3410 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3411 copy.dataTimeout = _defaultDataTimeout; 3412 return copy; 3413 } 3414 3415 private void initialize() 3416 { 3417 p.curl.initialize(); 3418 p.encoding = "ISO-8859-1"; 3419 dataTimeout = _defaultDataTimeout; 3420 } 3421 3422 /** 3423 Performs the ftp request as it has been configured. 3424 3425 After a FTP client has been setup and possibly assigned callbacks the $(D 3426 perform()) method will start performing the actual communication with the 3427 server. 3428 3429 Params: 3430 throwOnError = whether to throw an exception or return a CurlCode on error 3431 */ 3432 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3433 { 3434 return p.curl.perform(throwOnError); 3435 } 3436 3437 /// The URL to specify the location of the resource. 3438 @property void url(const(char)[] url) 3439 { 3440 import std.algorithm.searching : startsWith; 3441 import std.uni : toLower; 3442 3443 if (!startsWith(url.toLower(), "ftp://", "ftps://")) 3444 url = "ftp://" ~ url; 3445 p.curl.set(CurlOption.url, url); 3446 } 3447 3448 // This is a workaround for mixed in content not having its 3449 // docs mixed in. 3450 version (StdDdoc) 3451 { 3452 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to 3453 /// pause a request 3454 alias requestPause = CurlReadFunc.pause; 3455 3456 /// Value to return from onSend delegate in order to abort a request 3457 alias requestAbort = CurlReadFunc.abort; 3458 3459 /** 3460 True if the instance is stopped. A stopped instance is not usable. 3461 */ 3462 @property bool isStopped(); 3463 3464 /// Stop and invalidate this instance. 3465 void shutdown(); 3466 3467 /** Set verbose. 3468 This will print request information to stderr. 3469 */ 3470 @property void verbose(bool on); 3471 3472 // Connection settings 3473 3474 /// Set timeout for activity on connection. 3475 @property void dataTimeout(Duration d); 3476 3477 /** Set maximum time an operation is allowed to take. 3478 This includes dns resolution, connecting, data transfer, etc. 3479 */ 3480 @property void operationTimeout(Duration d); 3481 3482 /// Set timeout for connecting. 3483 @property void connectTimeout(Duration d); 3484 3485 // Network settings 3486 3487 /** Proxy 3488 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3489 */ 3490 @property void proxy(const(char)[] host); 3491 3492 /** Proxy port 3493 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3494 */ 3495 @property void proxyPort(ushort port); 3496 3497 /// Type of proxy 3498 alias CurlProxy = etc.c.curl.CurlProxy; 3499 3500 /** Proxy type 3501 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3502 */ 3503 @property void proxyType(CurlProxy type); 3504 3505 /// DNS lookup timeout. 3506 @property void dnsTimeout(Duration d); 3507 3508 /** 3509 * The network interface to use in form of the the IP of the interface. 3510 * 3511 * Example: 3512 * ---- 3513 * theprotocol.netInterface = "192.168.1.32"; 3514 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3515 * ---- 3516 * 3517 * See: $(REF InternetAddress, std,socket) 3518 */ 3519 @property void netInterface(const(char)[] i); 3520 3521 /// ditto 3522 @property void netInterface(const(ubyte)[4] i); 3523 3524 /// ditto 3525 @property void netInterface(InternetAddress i); 3526 3527 /** 3528 Set the local outgoing port to use. 3529 Params: 3530 port = the first outgoing port number to try and use 3531 */ 3532 @property void localPort(ushort port); 3533 3534 /** 3535 Set the local outgoing port range to use. 3536 This can be used together with the localPort property. 3537 Params: 3538 range = if the first port is occupied then try this many 3539 port number forwards 3540 */ 3541 @property void localPortRange(ushort range); 3542 3543 /** Set the tcp no-delay socket option on or off. 3544 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3545 */ 3546 @property void tcpNoDelay(bool on); 3547 3548 // Authentication settings 3549 3550 /** 3551 Set the user name, password and optionally domain for authentication 3552 purposes. 3553 3554 Some protocols may need authentication in some cases. Use this 3555 function to provide credentials. 3556 3557 Params: 3558 username = the username 3559 password = the password 3560 domain = used for NTLM authentication only and is set to the NTLM domain 3561 name 3562 */ 3563 void setAuthentication(const(char)[] username, const(char)[] password, 3564 const(char)[] domain = ""); 3565 3566 /** 3567 Set the user name and password for proxy authentication. 3568 3569 Params: 3570 username = the username 3571 password = the password 3572 */ 3573 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3574 3575 /** 3576 * The event handler that gets called when data is needed for sending. The 3577 * length of the $(D void[]) specifies the maximum number of bytes that can 3578 * be sent. 3579 * 3580 * Returns: 3581 * The callback returns the number of elements in the buffer that have been 3582 * filled and are ready to send. 3583 * The special value $(D .abortRequest) can be returned in order to abort the 3584 * current request. 3585 * The special value $(D .pauseRequest) can be returned in order to pause the 3586 * current request. 3587 * 3588 */ 3589 @property void onSend(size_t delegate(void[]) callback); 3590 3591 /** 3592 * The event handler that receives incoming data. Be sure to copy the 3593 * incoming ubyte[] since it is not guaranteed to be valid after the 3594 * callback returns. 3595 * 3596 * Returns: 3597 * The callback returns the incoming bytes read. If not the entire array is 3598 * the request will abort. 3599 * The special value .pauseRequest can be returned in order to pause the 3600 * current request. 3601 * 3602 */ 3603 @property void onReceive(size_t delegate(ubyte[]) callback); 3604 3605 /** 3606 * The event handler that gets called to inform of upload/download progress. 3607 * 3608 * Callback_parameters: 3609 * $(CALLBACK_PARAMS) 3610 * 3611 * Callback_returns: 3612 * Return 0 from the callback to signal success, return non-zero to 3613 * abort transfer. 3614 */ 3615 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 3616 size_t ulTotal, size_t ulNow) callback); 3617 } 3618 3619 /** Clear all commands send to ftp server. 3620 */ 3621 void clearCommands() 3622 { 3623 if (p.commands !is null) 3624 Curl.curl.slist_free_all(p.commands); 3625 p.commands = null; 3626 p.curl.clear(CurlOption.postquote); 3627 } 3628 3629 /** Add a command to send to ftp server. 3630 * 3631 * There is no remove command functionality. Do a $(LREF clearCommands) and 3632 * set the needed commands instead. 3633 * 3634 * Example: 3635 * --- 3636 * import std.net.curl; 3637 * auto client = FTP(); 3638 * client.addCommand("RNFR my_file.txt"); 3639 * client.addCommand("RNTO my_renamed_file.txt"); 3640 * upload("my_file.txt", "ftp.digitalmars.com", client); 3641 * --- 3642 */ 3643 void addCommand(const(char)[] command) 3644 { 3645 p.commands = Curl.curl.slist_append(p.commands, 3646 command.tempCString().buffPtr); 3647 p.curl.set(CurlOption.postquote, p.commands); 3648 } 3649 3650 /// Connection encoding. Defaults to ISO-8859-1. 3651 @property void encoding(string name) 3652 { 3653 p.encoding = name; 3654 } 3655 3656 /// ditto 3657 @property string encoding() 3658 { 3659 return p.encoding; 3660 } 3661 3662 /** 3663 The content length in bytes of the ftp data. 3664 */ 3665 @property void contentLength(ulong len) 3666 { 3667 import std.conv : to; 3668 p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len)); 3669 } 3670 3671 /** 3672 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 3673 * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok). 3674 * 3675 * Params: 3676 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 3677 * The values are: 3678 * $(D etc.c.curl.CurlInfo.namelookup_time), 3679 * $(D etc.c.curl.CurlInfo.connect_time), 3680 * $(D etc.c.curl.CurlInfo.pretransfer_time), 3681 * $(D etc.c.curl.CurlInfo.starttransfer_time), 3682 * $(D etc.c.curl.CurlInfo.redirect_time), 3683 * $(D etc.c.curl.CurlInfo.appconnect_time), 3684 * $(D etc.c.curl.CurlInfo.total_time). 3685 * val = the actual value of the inquired timing. 3686 * 3687 * Returns: 3688 * The return code of the operation. The value stored in val 3689 * should be used only if the return value is $(D etc.c.curl.CurlInfo.ok). 3690 * 3691 * Example: 3692 * --- 3693 * import std.net.curl; 3694 * import etc.c.curl : CurlError, CurlInfo; 3695 * 3696 * auto client = FTP(); 3697 * client.addCommand("RNFR my_file.txt"); 3698 * client.addCommand("RNTO my_renamed_file.txt"); 3699 * upload("my_file.txt", "ftp.digitalmars.com", client); 3700 * 3701 * double val; 3702 * CurlCode code; 3703 * 3704 * code = http.getTiming(CurlInfo.namelookup_time, val); 3705 * assert(code == CurlError.ok); 3706 * --- 3707 */ 3708 CurlCode getTiming(CurlInfo timing, ref double val) 3709 { 3710 return p.curl.getTiming(timing, val); 3711 } 3712 3713 @system unittest 3714 { 3715 auto client = FTP(); 3716 3717 double val; 3718 CurlCode code; 3719 3720 code = client.getTiming(CurlInfo.total_time, val); 3721 assert(code == CurlError.ok); 3722 code = client.getTiming(CurlInfo.namelookup_time, val); 3723 assert(code == CurlError.ok); 3724 code = client.getTiming(CurlInfo.connect_time, val); 3725 assert(code == CurlError.ok); 3726 code = client.getTiming(CurlInfo.pretransfer_time, val); 3727 assert(code == CurlError.ok); 3728 code = client.getTiming(CurlInfo.starttransfer_time, val); 3729 assert(code == CurlError.ok); 3730 code = client.getTiming(CurlInfo.redirect_time, val); 3731 assert(code == CurlError.ok); 3732 code = client.getTiming(CurlInfo.appconnect_time, val); 3733 assert(code == CurlError.ok); 3734 } 3735} 3736 3737/** 3738 * Basic SMTP protocol support. 3739 * 3740 * Example: 3741 * --- 3742 * import std.net.curl; 3743 * 3744 * // Send an email with SMTPS 3745 * auto smtp = SMTP("smtps://smtp.gmail.com"); 3746 * smtp.setAuthentication("from.addr@gmail.com", "password"); 3747 * smtp.mailTo = ["<to.addr@gmail.com>"]; 3748 * smtp.mailFrom = "<from.addr@gmail.com>"; 3749 * smtp.message = "Example Message"; 3750 * smtp.perform(); 3751 * --- 3752 * 3753 * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821) 3754 */ 3755struct SMTP 3756{ 3757 mixin Protocol; 3758 3759 private struct Impl 3760 { 3761 ~this() 3762 { 3763 if (curl.handle !is null) // work around RefCounted/emplace bug 3764 curl.shutdown(); 3765 } 3766 Curl curl; 3767 3768 @property void message(string msg) 3769 { 3770 import std.algorithm.comparison : min; 3771 3772 auto _message = msg; 3773 /** 3774 This delegate reads the message text and copies it. 3775 */ 3776 curl.onSend = delegate size_t(void[] data) 3777 { 3778 if (!msg.length) return 0; 3779 size_t to_copy = min(data.length, _message.length); 3780 data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy]; 3781 _message = _message[to_copy..$]; 3782 return to_copy; 3783 }; 3784 } 3785 } 3786 3787 private RefCounted!Impl p; 3788 3789 /** 3790 Sets to the URL of the SMTP server. 3791 */ 3792 static SMTP opCall(const(char)[] url) 3793 { 3794 SMTP smtp; 3795 smtp.initialize(); 3796 smtp.url = url; 3797 return smtp; 3798 } 3799 3800 /// 3801 static SMTP opCall() 3802 { 3803 SMTP smtp; 3804 smtp.initialize(); 3805 return smtp; 3806 } 3807 3808 /+ TODO: The other structs have this function. 3809 SMTP dup() 3810 { 3811 SMTP copy = SMTP(); 3812 copy.initialize(); 3813 copy.p.encoding = p.encoding; 3814 copy.p.curl = p.curl.dup(); 3815 curl_slist* cur = p.commands; 3816 curl_slist* newlist = null; 3817 while (cur) 3818 { 3819 newlist = Curl.curl.slist_append(newlist, cur.data); 3820 cur = cur.next; 3821 } 3822 copy.p.commands = newlist; 3823 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3824 copy.dataTimeout = _defaultDataTimeout; 3825 return copy; 3826 } 3827 +/ 3828 3829 /** 3830 Performs the request as configured. 3831 Params: 3832 throwOnError = whether to throw an exception or return a CurlCode on error 3833 */ 3834 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3835 { 3836 return p.curl.perform(throwOnError); 3837 } 3838 3839 /// The URL to specify the location of the resource. 3840 @property void url(const(char)[] url) 3841 { 3842 import std.algorithm.searching : startsWith; 3843 import std.uni : toLower; 3844 3845 auto lowered = url.toLower(); 3846 3847 if (lowered.startsWith("smtps://")) 3848 { 3849 p.curl.set(CurlOption.use_ssl, CurlUseSSL.all); 3850 } 3851 else 3852 { 3853 enforce!CurlException(lowered.startsWith("smtp://"), 3854 "The url must be for the smtp protocol."); 3855 } 3856 p.curl.set(CurlOption.url, url); 3857 } 3858 3859 private void initialize() 3860 { 3861 p.curl.initialize(); 3862 p.curl.set(CurlOption.upload, 1L); 3863 dataTimeout = _defaultDataTimeout; 3864 verifyPeer = true; 3865 verifyHost = true; 3866 } 3867 3868 // This is a workaround for mixed in content not having its 3869 // docs mixed in. 3870 version (StdDdoc) 3871 { 3872 /// Value to return from $(D onSend)/$(D onReceive) delegates in order to 3873 /// pause a request 3874 alias requestPause = CurlReadFunc.pause; 3875 3876 /// Value to return from onSend delegate in order to abort a request 3877 alias requestAbort = CurlReadFunc.abort; 3878 3879 /** 3880 True if the instance is stopped. A stopped instance is not usable. 3881 */ 3882 @property bool isStopped(); 3883 3884 /// Stop and invalidate this instance. 3885 void shutdown(); 3886 3887 /** Set verbose. 3888 This will print request information to stderr. 3889 */ 3890 @property void verbose(bool on); 3891 3892 // Connection settings 3893 3894 /// Set timeout for activity on connection. 3895 @property void dataTimeout(Duration d); 3896 3897 /** Set maximum time an operation is allowed to take. 3898 This includes dns resolution, connecting, data transfer, etc. 3899 */ 3900 @property void operationTimeout(Duration d); 3901 3902 /// Set timeout for connecting. 3903 @property void connectTimeout(Duration d); 3904 3905 // Network settings 3906 3907 /** Proxy 3908 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3909 */ 3910 @property void proxy(const(char)[] host); 3911 3912 /** Proxy port 3913 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3914 */ 3915 @property void proxyPort(ushort port); 3916 3917 /// Type of proxy 3918 alias CurlProxy = etc.c.curl.CurlProxy; 3919 3920 /** Proxy type 3921 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3922 */ 3923 @property void proxyType(CurlProxy type); 3924 3925 /// DNS lookup timeout. 3926 @property void dnsTimeout(Duration d); 3927 3928 /** 3929 * The network interface to use in form of the the IP of the interface. 3930 * 3931 * Example: 3932 * ---- 3933 * theprotocol.netInterface = "192.168.1.32"; 3934 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3935 * ---- 3936 * 3937 * See: $(REF InternetAddress, std,socket) 3938 */ 3939 @property void netInterface(const(char)[] i); 3940 3941 /// ditto 3942 @property void netInterface(const(ubyte)[4] i); 3943 3944 /// ditto 3945 @property void netInterface(InternetAddress i); 3946 3947 /** 3948 Set the local outgoing port to use. 3949 Params: 3950 port = the first outgoing port number to try and use 3951 */ 3952 @property void localPort(ushort port); 3953 3954 /** 3955 Set the local outgoing port range to use. 3956 This can be used together with the localPort property. 3957 Params: 3958 range = if the first port is occupied then try this many 3959 port number forwards 3960 */ 3961 @property void localPortRange(ushort range); 3962 3963 /** Set the tcp no-delay socket option on or off. 3964 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3965 */ 3966 @property void tcpNoDelay(bool on); 3967 3968 // Authentication settings 3969 3970 /** 3971 Set the user name, password and optionally domain for authentication 3972 purposes. 3973 3974 Some protocols may need authentication in some cases. Use this 3975 function to provide credentials. 3976 3977 Params: 3978 username = the username 3979 password = the password 3980 domain = used for NTLM authentication only and is set to the NTLM domain 3981 name 3982 */ 3983 void setAuthentication(const(char)[] username, const(char)[] password, 3984 const(char)[] domain = ""); 3985 3986 /** 3987 Set the user name and password for proxy authentication. 3988 3989 Params: 3990 username = the username 3991 password = the password 3992 */ 3993 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3994 3995 /** 3996 * The event handler that gets called when data is needed for sending. The 3997 * length of the $(D void[]) specifies the maximum number of bytes that can 3998 * be sent. 3999 * 4000 * Returns: 4001 * The callback returns the number of elements in the buffer that have been 4002 * filled and are ready to send. 4003 * The special value $(D .abortRequest) can be returned in order to abort the 4004 * current request. 4005 * The special value $(D .pauseRequest) can be returned in order to pause the 4006 * current request. 4007 */ 4008 @property void onSend(size_t delegate(void[]) callback); 4009 4010 /** 4011 * The event handler that receives incoming data. Be sure to copy the 4012 * incoming ubyte[] since it is not guaranteed to be valid after the 4013 * callback returns. 4014 * 4015 * Returns: 4016 * The callback returns the incoming bytes read. If not the entire array is 4017 * the request will abort. 4018 * The special value .pauseRequest can be returned in order to pause the 4019 * current request. 4020 */ 4021 @property void onReceive(size_t delegate(ubyte[]) callback); 4022 4023 /** 4024 * The event handler that gets called to inform of upload/download progress. 4025 * 4026 * Callback_parameters: 4027 * $(CALLBACK_PARAMS) 4028 * 4029 * Callback_returns: 4030 * Return 0 from the callback to signal success, return non-zero to 4031 * abort transfer. 4032 */ 4033 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 4034 size_t ulTotal, size_t ulNow) callback); 4035 } 4036 4037 /** 4038 Setter for the sender's email address. 4039 */ 4040 @property void mailFrom()(const(char)[] sender) 4041 { 4042 assert(!sender.empty, "Sender must not be empty"); 4043 p.curl.set(CurlOption.mail_from, sender); 4044 } 4045 4046 /** 4047 Setter for the recipient email addresses. 4048 */ 4049 void mailTo()(const(char)[][] recipients...) 4050 { 4051 assert(!recipients.empty, "Recipient must not be empty"); 4052 curl_slist* recipients_list = null; 4053 foreach (recipient; recipients) 4054 { 4055 recipients_list = 4056 Curl.curl.slist_append(recipients_list, 4057 recipient.tempCString().buffPtr); 4058 } 4059 p.curl.set(CurlOption.mail_rcpt, recipients_list); 4060 } 4061 4062 /** 4063 Sets the message body text. 4064 */ 4065 4066 @property void message(string msg) 4067 { 4068 p.message = msg; 4069 } 4070} 4071 4072/++ 4073 Exception thrown on errors in std.net.curl functions. 4074+/ 4075class CurlException : Exception 4076{ 4077 /++ 4078 Params: 4079 msg = The message for the exception. 4080 file = The file where the exception occurred. 4081 line = The line number where the exception occurred. 4082 next = The previous exception in the chain of exceptions, if any. 4083 +/ 4084 @safe pure nothrow 4085 this(string msg, 4086 string file = __FILE__, 4087 size_t line = __LINE__, 4088 Throwable next = null) 4089 { 4090 super(msg, file, line, next); 4091 } 4092} 4093 4094/++ 4095 Exception thrown on timeout errors in std.net.curl functions. 4096+/ 4097class CurlTimeoutException : CurlException 4098{ 4099 /++ 4100 Params: 4101 msg = The message for the exception. 4102 file = The file where the exception occurred. 4103 line = The line number where the exception occurred. 4104 next = The previous exception in the chain of exceptions, if any. 4105 +/ 4106 @safe pure nothrow 4107 this(string msg, 4108 string file = __FILE__, 4109 size_t line = __LINE__, 4110 Throwable next = null) 4111 { 4112 super(msg, file, line, next); 4113 } 4114} 4115 4116/++ 4117 Exception thrown on HTTP request failures, e.g. 404 Not Found. 4118+/ 4119class HTTPStatusException : CurlException 4120{ 4121 /++ 4122 Params: 4123 status = The HTTP status code. 4124 msg = The message for the exception. 4125 file = The file where the exception occurred. 4126 line = The line number where the exception occurred. 4127 next = The previous exception in the chain of exceptions, if any. 4128 +/ 4129 @safe pure nothrow 4130 this(int status, 4131 string msg, 4132 string file = __FILE__, 4133 size_t line = __LINE__, 4134 Throwable next = null) 4135 { 4136 super(msg, file, line, next); 4137 this.status = status; 4138 } 4139 4140 immutable int status; /// The HTTP status code 4141} 4142 4143/// Equal to $(REF CURLcode, etc,c,curl) 4144alias CurlCode = CURLcode; 4145 4146import std.typecons : Flag, Yes, No; 4147/// Flag to specify whether or not an exception is thrown on error. 4148alias ThrowOnError = Flag!"throwOnError"; 4149 4150private struct CurlAPI 4151{ 4152 static struct API 4153 { 4154 extern(C): 4155 import core.stdc.config : c_long; 4156 CURLcode function(c_long flags) global_init; 4157 void function() global_cleanup; 4158 curl_version_info_data * function(CURLversion) version_info; 4159 CURL* function() easy_init; 4160 CURLcode function(CURL *curl, CURLoption option,...) easy_setopt; 4161 CURLcode function(CURL *curl) easy_perform; 4162 CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo; 4163 CURL* function(CURL *curl) easy_duphandle; 4164 char* function(CURLcode) easy_strerror; 4165 CURLcode function(CURL *handle, int bitmask) easy_pause; 4166 void function(CURL *curl) easy_cleanup; 4167 curl_slist* function(curl_slist *, char *) slist_append; 4168 void function(curl_slist *) slist_free_all; 4169 } 4170 __gshared API _api; 4171 __gshared void* _handle; 4172 4173 static ref API instance() @property 4174 { 4175 import std.concurrency : initOnce; 4176 initOnce!_handle(loadAPI()); 4177 return _api; 4178 } 4179 4180 static void* loadAPI() 4181 { 4182 version (Posix) 4183 { 4184 import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY; 4185 alias loadSym = dlsym; 4186 } 4187 else version (Windows) 4188 { 4189 import core.sys.windows.windows : GetProcAddress, GetModuleHandleA, 4190 LoadLibraryA; 4191 alias loadSym = GetProcAddress; 4192 } 4193 else 4194 static assert(0, "unimplemented"); 4195 4196 void* handle; 4197 version (Posix) 4198 handle = dlopen(null, RTLD_LAZY); 4199 else version (Windows) 4200 handle = GetModuleHandleA(null); 4201 assert(handle !is null); 4202 4203 // try to load curl from the executable to allow static linking 4204 if (loadSym(handle, "curl_global_init") is null) 4205 { 4206 import std.format : format; 4207 version (Posix) 4208 dlclose(handle); 4209 4210 version (OSX) 4211 static immutable names = ["libcurl.4.dylib"]; 4212 else version (Posix) 4213 { 4214 static immutable names = ["libcurl.so", "libcurl.so.4", 4215 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"]; 4216 } 4217 else version (Windows) 4218 static immutable names = ["libcurl.dll", "curl.dll"]; 4219 4220 foreach (name; names) 4221 { 4222 version (Posix) 4223 handle = dlopen(name.ptr, RTLD_LAZY); 4224 else version (Windows) 4225 handle = LoadLibraryA(name.ptr); 4226 if (handle !is null) break; 4227 } 4228 4229 enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); 4230 } 4231 4232 foreach (i, FP; typeof(API.tupleof)) 4233 { 4234 enum name = __traits(identifier, _api.tupleof[i]); 4235 auto p = enforce!CurlException(loadSym(handle, "curl_"~name), 4236 "Couldn't load curl_"~name~" from libcurl."); 4237 _api.tupleof[i] = cast(FP) p; 4238 } 4239 4240 enforce!CurlException(!_api.global_init(CurlGlobal.all), 4241 "Failed to initialize libcurl"); 4242 4243 static extern(C) void cleanup() 4244 { 4245 if (_handle is null) return; 4246 _api.global_cleanup(); 4247 version (Posix) 4248 { 4249 import core.sys.posix.dlfcn : dlclose; 4250 dlclose(_handle); 4251 } 4252 else version (Windows) 4253 { 4254 import core.sys.windows.windows : FreeLibrary; 4255 FreeLibrary(_handle); 4256 } 4257 else 4258 static assert(0, "unimplemented"); 4259 _api = API.init; 4260 _handle = null; 4261 } 4262 4263 import core.stdc.stdlib : atexit; 4264 atexit(&cleanup); 4265 4266 return handle; 4267 } 4268} 4269 4270/** 4271 Wrapper to provide a better interface to libcurl than using the plain C API. 4272 It is recommended to use the $(D HTTP)/$(D FTP) etc. structs instead unless 4273 raw access to libcurl is needed. 4274 4275 Warning: This struct uses interior pointers for callbacks. Only allocate it 4276 on the stack if you never move or copy it. This also means passing by reference 4277 when passing Curl to other functions. Otherwise always allocate on 4278 the heap. 4279*/ 4280struct Curl 4281{ 4282 alias OutData = void[]; 4283 alias InData = ubyte[]; 4284 private bool _stopped; 4285 4286 private static auto ref curl() @property { return CurlAPI.instance; } 4287 4288 // A handle should not be used by two threads simultaneously 4289 private CURL* handle; 4290 4291 // May also return $(D CURL_READFUNC_ABORT) or $(D CURL_READFUNC_PAUSE) 4292 private size_t delegate(OutData) _onSend; 4293 private size_t delegate(InData) _onReceive; 4294 private void delegate(in char[]) _onReceiveHeader; 4295 private CurlSeek delegate(long,CurlSeekPos) _onSeek; 4296 private int delegate(curl_socket_t,CurlSockType) _onSocketOption; 4297 private int delegate(size_t dltotal, size_t dlnow, 4298 size_t ultotal, size_t ulnow) _onProgress; 4299 4300 alias requestPause = CurlReadFunc.pause; 4301 alias requestAbort = CurlReadFunc.abort; 4302 4303 /** 4304 Initialize the instance by creating a working curl handle. 4305 */ 4306 void initialize() 4307 { 4308 enforce!CurlException(!handle, "Curl instance already initialized"); 4309 handle = curl.easy_init(); 4310 enforce!CurlException(handle, "Curl instance couldn't be initialized"); 4311 _stopped = false; 4312 set(CurlOption.nosignal, 1); 4313 } 4314 4315 /// 4316 @property bool stopped() const 4317 { 4318 return _stopped; 4319 } 4320 4321 /** 4322 Duplicate this handle. 4323 4324 The new handle will have all options set as the one it was duplicated 4325 from. An exception to this is that all options that cannot be shared 4326 across threads are reset thereby making it safe to use the duplicate 4327 in a new thread. 4328 */ 4329 Curl dup() 4330 { 4331 Curl copy; 4332 copy.handle = curl.easy_duphandle(handle); 4333 copy._stopped = false; 4334 4335 with (CurlOption) { 4336 auto tt = AliasSeq!(file, writefunction, writeheader, 4337 headerfunction, infile, readfunction, ioctldata, ioctlfunction, 4338 seekdata, seekfunction, sockoptdata, sockoptfunction, 4339 opensocketdata, opensocketfunction, progressdata, 4340 progressfunction, debugdata, debugfunction, interleavedata, 4341 interleavefunction, chunk_data, chunk_bgn_function, 4342 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields); 4343 4344 foreach (option; tt) 4345 copy.clear(option); 4346 } 4347 4348 // The options are only supported by libcurl when it has been built 4349 // against certain versions of OpenSSL - if your libcurl uses an old 4350 // OpenSSL, or uses an entirely different SSL engine, attempting to 4351 // clear these normally will raise an exception 4352 copy.clearIfSupported(CurlOption.ssl_ctx_function); 4353 copy.clearIfSupported(CurlOption.ssh_keydata); 4354 4355 // Enable for curl version > 7.21.7 4356 static if (LIBCURL_VERSION_MAJOR >= 7 && 4357 LIBCURL_VERSION_MINOR >= 21 && 4358 LIBCURL_VERSION_PATCH >= 7) 4359 { 4360 copy.clear(CurlOption.closesocketdata); 4361 copy.clear(CurlOption.closesocketfunction); 4362 } 4363 4364 copy.set(CurlOption.nosignal, 1); 4365 4366 // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared 4367 // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared 4368 4369 /* 4370 Allow sharing of conv functions 4371 copy.clear(CurlOption.conv_to_network_function); 4372 copy.clear(CurlOption.conv_from_network_function); 4373 copy.clear(CurlOption.conv_from_utf8_function); 4374 */ 4375 4376 return copy; 4377 } 4378 4379 private void _check(CurlCode code) 4380 { 4381 enforce!CurlTimeoutException(code != CurlError.operation_timedout, 4382 errorString(code)); 4383 4384 enforce!CurlException(code == CurlError.ok, 4385 errorString(code)); 4386 } 4387 4388 private string errorString(CurlCode code) 4389 { 4390 import core.stdc.string : strlen; 4391 import std.format : format; 4392 4393 auto msgZ = curl.easy_strerror(code); 4394 // doing the following (instead of just using std.conv.to!string) avoids 1 allocation 4395 return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle); 4396 } 4397 4398 private void throwOnStopped(string message = null) 4399 { 4400 auto def = "Curl instance called after being cleaned up"; 4401 enforce!CurlException(!stopped, 4402 message == null ? def : message); 4403 } 4404 4405 /** 4406 Stop and invalidate this curl instance. 4407 Warning: Do not call this from inside a callback handler e.g. $(D onReceive). 4408 */ 4409 void shutdown() 4410 { 4411 throwOnStopped(); 4412 _stopped = true; 4413 curl.easy_cleanup(this.handle); 4414 this.handle = null; 4415 } 4416 4417 /** 4418 Pausing and continuing transfers. 4419 */ 4420 void pause(bool sendingPaused, bool receivingPaused) 4421 { 4422 throwOnStopped(); 4423 _check(curl.easy_pause(this.handle, 4424 (sendingPaused ? CurlPause.send_cont : CurlPause.send) | 4425 (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); 4426 } 4427 4428 /** 4429 Set a string curl option. 4430 Params: 4431 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4432 value = The string 4433 */ 4434 void set(CurlOption option, const(char)[] value) 4435 { 4436 throwOnStopped(); 4437 _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr)); 4438 } 4439 4440 /** 4441 Set a long curl option. 4442 Params: 4443 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4444 value = The long 4445 */ 4446 void set(CurlOption option, long value) 4447 { 4448 throwOnStopped(); 4449 _check(curl.easy_setopt(this.handle, option, value)); 4450 } 4451 4452 /** 4453 Set a void* curl option. 4454 Params: 4455 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4456 value = The pointer 4457 */ 4458 void set(CurlOption option, void* value) 4459 { 4460 throwOnStopped(); 4461 _check(curl.easy_setopt(this.handle, option, value)); 4462 } 4463 4464 /** 4465 Clear a pointer option. 4466 Params: 4467 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4468 */ 4469 void clear(CurlOption option) 4470 { 4471 throwOnStopped(); 4472 _check(curl.easy_setopt(this.handle, option, null)); 4473 } 4474 4475 /** 4476 Clear a pointer option. Does not raise an exception if the underlying 4477 libcurl does not support the option. Use sparingly. 4478 Params: 4479 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4480 */ 4481 void clearIfSupported(CurlOption option) 4482 { 4483 throwOnStopped(); 4484 auto rval = curl.easy_setopt(this.handle, option, null); 4485 if (rval != CurlError.unknown_option && rval != CurlError.not_built_in) 4486 _check(rval); 4487 } 4488 4489 /** 4490 perform the curl request by doing the HTTP,FTP etc. as it has 4491 been setup beforehand. 4492 4493 Params: 4494 throwOnError = whether to throw an exception or return a CurlCode on error 4495 */ 4496 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 4497 { 4498 throwOnStopped(); 4499 CurlCode code = curl.easy_perform(this.handle); 4500 if (throwOnError) 4501 _check(code); 4502 return code; 4503 } 4504 4505 /** 4506 Get the various timings like name lookup time, total time, connect time etc. 4507 The timed category is passed through the timing parameter while the timing 4508 value is stored at val. The value is usable only if res is equal to 4509 $(D etc.c.curl.CurlError.ok). 4510 */ 4511 CurlCode getTiming(CurlInfo timing, ref double val) 4512 { 4513 CurlCode code; 4514 code = curl.easy_getinfo(handle, timing, &val); 4515 return code; 4516 } 4517 4518 /** 4519 * The event handler that receives incoming data. 4520 * 4521 * Params: 4522 * callback = the callback that receives the $(D ubyte[]) data. 4523 * Be sure to copy the incoming data and not store 4524 * a slice. 4525 * 4526 * Returns: 4527 * The callback returns the incoming bytes read. If not the entire array is 4528 * the request will abort. 4529 * The special value HTTP.pauseRequest can be returned in order to pause the 4530 * current request. 4531 * 4532 * Example: 4533 * ---- 4534 * import std.net.curl, std.stdio; 4535 * Curl curl; 4536 * curl.initialize(); 4537 * curl.set(CurlOption.url, "http://dlang.org"); 4538 * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; 4539 * curl.perform(); 4540 * ---- 4541 */ 4542 @property void onReceive(size_t delegate(InData) callback) 4543 { 4544 _onReceive = (InData id) 4545 { 4546 throwOnStopped("Receive callback called on cleaned up Curl instance"); 4547 return callback(id); 4548 }; 4549 set(CurlOption.file, cast(void*) &this); 4550 set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback); 4551 } 4552 4553 /** 4554 * The event handler that receives incoming headers for protocols 4555 * that uses headers. 4556 * 4557 * Params: 4558 * callback = the callback that receives the header string. 4559 * Make sure the callback copies the incoming params if 4560 * it needs to store it because they are references into 4561 * the backend and may very likely change. 4562 * 4563 * Example: 4564 * ---- 4565 * import std.net.curl, std.stdio; 4566 * Curl curl; 4567 * curl.initialize(); 4568 * curl.set(CurlOption.url, "http://dlang.org"); 4569 * curl.onReceiveHeader = (in char[] header) { writeln(header); }; 4570 * curl.perform(); 4571 * ---- 4572 */ 4573 @property void onReceiveHeader(void delegate(in char[]) callback) 4574 { 4575 _onReceiveHeader = (in char[] od) 4576 { 4577 throwOnStopped("Receive header callback called on "~ 4578 "cleaned up Curl instance"); 4579 callback(od); 4580 }; 4581 set(CurlOption.writeheader, cast(void*) &this); 4582 set(CurlOption.headerfunction, 4583 cast(void*) &Curl._receiveHeaderCallback); 4584 } 4585 4586 /** 4587 * The event handler that gets called when data is needed for sending. 4588 * 4589 * Params: 4590 * callback = the callback that has a $(D void[]) buffer to be filled 4591 * 4592 * Returns: 4593 * The callback returns the number of elements in the buffer that have been 4594 * filled and are ready to send. 4595 * The special value $(D Curl.abortRequest) can be returned in 4596 * order to abort the current request. 4597 * The special value $(D Curl.pauseRequest) can be returned in order to 4598 * pause the current request. 4599 * 4600 * Example: 4601 * ---- 4602 * import std.net.curl; 4603 * Curl curl; 4604 * curl.initialize(); 4605 * curl.set(CurlOption.url, "http://dlang.org"); 4606 * 4607 * string msg = "Hello world"; 4608 * curl.onSend = (void[] data) 4609 * { 4610 * auto m = cast(void[]) msg; 4611 * size_t length = m.length > data.length ? data.length : m.length; 4612 * if (length == 0) return 0; 4613 * data[0 .. length] = m[0 .. length]; 4614 * msg = msg[length..$]; 4615 * return length; 4616 * }; 4617 * curl.perform(); 4618 * ---- 4619 */ 4620 @property void onSend(size_t delegate(OutData) callback) 4621 { 4622 _onSend = (OutData od) 4623 { 4624 throwOnStopped("Send callback called on cleaned up Curl instance"); 4625 return callback(od); 4626 }; 4627 set(CurlOption.infile, cast(void*) &this); 4628 set(CurlOption.readfunction, cast(void*) &Curl._sendCallback); 4629 } 4630 4631 /** 4632 * The event handler that gets called when the curl backend needs to seek 4633 * the data to be sent. 4634 * 4635 * Params: 4636 * callback = the callback that receives a seek offset and a seek position 4637 * $(REF CurlSeekPos, etc,c,curl) 4638 * 4639 * Returns: 4640 * The callback returns the success state of the seeking 4641 * $(REF CurlSeek, etc,c,curl) 4642 * 4643 * Example: 4644 * ---- 4645 * import std.net.curl; 4646 * Curl curl; 4647 * curl.initialize(); 4648 * curl.set(CurlOption.url, "http://dlang.org"); 4649 * curl.onSeek = (long p, CurlSeekPos sp) 4650 * { 4651 * return CurlSeek.cantseek; 4652 * }; 4653 * curl.perform(); 4654 * ---- 4655 */ 4656 @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback) 4657 { 4658 _onSeek = (long ofs, CurlSeekPos sp) 4659 { 4660 throwOnStopped("Seek callback called on cleaned up Curl instance"); 4661 return callback(ofs, sp); 4662 }; 4663 set(CurlOption.seekdata, cast(void*) &this); 4664 set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback); 4665 } 4666 4667 /** 4668 * The event handler that gets called when the net socket has been created 4669 * but a $(D connect()) call has not yet been done. This makes it possible to set 4670 * misc. socket options. 4671 * 4672 * Params: 4673 * callback = the callback that receives the socket and socket type 4674 * $(REF CurlSockType, etc,c,curl) 4675 * 4676 * Returns: 4677 * Return 0 from the callback to signal success, return 1 to signal error 4678 * and make curl close the socket 4679 * 4680 * Example: 4681 * ---- 4682 * import std.net.curl; 4683 * Curl curl; 4684 * curl.initialize(); 4685 * curl.set(CurlOption.url, "http://dlang.org"); 4686 * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; 4687 * curl.perform(); 4688 * ---- 4689 */ 4690 @property void onSocketOption(int delegate(curl_socket_t, 4691 CurlSockType) callback) 4692 { 4693 _onSocketOption = (curl_socket_t sock, CurlSockType st) 4694 { 4695 throwOnStopped("Socket option callback called on "~ 4696 "cleaned up Curl instance"); 4697 return callback(sock, st); 4698 }; 4699 set(CurlOption.sockoptdata, cast(void*) &this); 4700 set(CurlOption.sockoptfunction, 4701 cast(void*) &Curl._socketOptionCallback); 4702 } 4703 4704 /** 4705 * The event handler that gets called to inform of upload/download progress. 4706 * 4707 * Params: 4708 * callback = the callback that receives the (total bytes to download, 4709 * currently downloaded bytes, total bytes to upload, currently uploaded 4710 * bytes). 4711 * 4712 * Returns: 4713 * Return 0 from the callback to signal success, return non-zero to abort 4714 * transfer 4715 * 4716 * Example: 4717 * ---- 4718 * import std.net.curl; 4719 * Curl curl; 4720 * curl.initialize(); 4721 * curl.set(CurlOption.url, "http://dlang.org"); 4722 * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t uln) 4723 * { 4724 * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); 4725 * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); 4726 * curl.perform(); 4727 * }; 4728 * ---- 4729 */ 4730 @property void onProgress(int delegate(size_t dlTotal, 4731 size_t dlNow, 4732 size_t ulTotal, 4733 size_t ulNow) callback) 4734 { 4735 _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) 4736 { 4737 throwOnStopped("Progress callback called on cleaned "~ 4738 "up Curl instance"); 4739 return callback(dlt, dln, ult, uln); 4740 }; 4741 set(CurlOption.noprogress, 0); 4742 set(CurlOption.progressdata, cast(void*) &this); 4743 set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback); 4744 } 4745 4746 // Internal C callbacks to register with libcurl 4747 extern (C) private static 4748 size_t _receiveCallback(const char* str, 4749 size_t size, size_t nmemb, void* ptr) 4750 { 4751 auto b = cast(Curl*) ptr; 4752 if (b._onReceive != null) 4753 return b._onReceive(cast(InData)(str[0 .. size*nmemb])); 4754 return size*nmemb; 4755 } 4756 4757 extern (C) private static 4758 size_t _receiveHeaderCallback(const char* str, 4759 size_t size, size_t nmemb, void* ptr) 4760 { 4761 import std.string : chomp; 4762 4763 auto b = cast(Curl*) ptr; 4764 auto s = str[0 .. size*nmemb].chomp(); 4765 if (b._onReceiveHeader != null) 4766 b._onReceiveHeader(s); 4767 4768 return size*nmemb; 4769 } 4770 4771 extern (C) private static 4772 size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr) 4773 { 4774 Curl* b = cast(Curl*) ptr; 4775 auto a = cast(void[]) str[0 .. size*nmemb]; 4776 if (b._onSend == null) 4777 return 0; 4778 return b._onSend(a); 4779 } 4780 4781 extern (C) private static 4782 int _seekCallback(void *ptr, curl_off_t offset, int origin) 4783 { 4784 auto b = cast(Curl*) ptr; 4785 if (b._onSeek == null) 4786 return CurlSeek.cantseek; 4787 4788 // origin: CurlSeekPos.set/current/end 4789 // return: CurlSeek.ok/fail/cantseek 4790 return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin); 4791 } 4792 4793 extern (C) private static 4794 int _socketOptionCallback(void *ptr, 4795 curl_socket_t curlfd, curlsocktype purpose) 4796 { 4797 auto b = cast(Curl*) ptr; 4798 if (b._onSocketOption == null) 4799 return 0; 4800 4801 // return: 0 ok, 1 fail 4802 return b._onSocketOption(curlfd, cast(CurlSockType) purpose); 4803 } 4804 4805 extern (C) private static 4806 int _progressCallback(void *ptr, 4807 double dltotal, double dlnow, 4808 double ultotal, double ulnow) 4809 { 4810 auto b = cast(Curl*) ptr; 4811 if (b._onProgress == null) 4812 return 0; 4813 4814 // return: 0 ok, 1 fail 4815 return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow, 4816 cast(size_t) ultotal, cast(size_t) ulnow); 4817 } 4818 4819} 4820 4821// Internal messages send between threads. 4822// The data is wrapped in this struct in order to ensure that 4823// other std.concurrency.receive calls does not pick up our messages 4824// by accident. 4825private struct CurlMessage(T) 4826{ 4827 public T data; 4828} 4829 4830private static CurlMessage!T curlMessage(T)(T data) 4831{ 4832 return CurlMessage!T(data); 4833} 4834 4835// Pool of to be used for reusing buffers 4836private struct Pool(Data) 4837{ 4838 private struct Entry 4839 { 4840 Data data; 4841 Entry* next; 4842 } 4843 private Entry* root; 4844 private Entry* freeList; 4845 4846 @safe @property bool empty() 4847 { 4848 return root == null; 4849 } 4850 4851 @safe nothrow void push(Data d) 4852 { 4853 if (freeList == null) 4854 { 4855 // Allocate new Entry since there is no one 4856 // available in the freeList 4857 freeList = new Entry; 4858 } 4859 freeList.data = d; 4860 Entry* oldroot = root; 4861 root = freeList; 4862 freeList = freeList.next; 4863 root.next = oldroot; 4864 } 4865 4866 @safe Data pop() 4867 { 4868 enforce!Exception(root != null, "pop() called on empty pool"); 4869 auto d = root.data; 4870 auto n = root.next; 4871 root.next = freeList; 4872 freeList = root; 4873 root = n; 4874 return d; 4875 } 4876} 4877 4878// Shared function for reading incoming chunks of data and 4879// sending the to a parent thread 4880private static size_t _receiveAsyncChunks(ubyte[] data, ref ubyte[] outdata, 4881 Pool!(ubyte[]) freeBuffers, 4882 ref ubyte[] buffer, Tid fromTid, 4883 ref bool aborted) 4884{ 4885 immutable datalen = data.length; 4886 4887 // Copy data to fill active buffer 4888 while (!data.empty) 4889 { 4890 4891 // Make sure a buffer is present 4892 while ( outdata.empty && freeBuffers.empty) 4893 { 4894 // Active buffer is invalid and there are no 4895 // available buffers in the pool. Wait for buffers 4896 // to return from main thread in order to reuse 4897 // them. 4898 receive((immutable(ubyte)[] buf) 4899 { 4900 buffer = cast(ubyte[]) buf; 4901 outdata = buffer[]; 4902 }, 4903 (bool flag) { aborted = true; } 4904 ); 4905 if (aborted) return cast(size_t) 0; 4906 } 4907 if (outdata.empty) 4908 { 4909 buffer = freeBuffers.pop(); 4910 outdata = buffer[]; 4911 } 4912 4913 // Copy data 4914 auto copyBytes = outdata.length < data.length ? 4915 outdata.length : data.length; 4916 4917 outdata[0 .. copyBytes] = data[0 .. copyBytes]; 4918 outdata = outdata[copyBytes..$]; 4919 data = data[copyBytes..$]; 4920 4921 if (outdata.empty) 4922 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 4923 } 4924 4925 return datalen; 4926} 4927 4928// ditto 4929private static void _finalizeAsyncChunks(ubyte[] outdata, ref ubyte[] buffer, 4930 Tid fromTid) 4931{ 4932 if (!outdata.empty) 4933 { 4934 // Resize the last buffer 4935 buffer.length = buffer.length - outdata.length; 4936 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 4937 } 4938} 4939 4940 4941// Shared function for reading incoming lines of data and sending the to a 4942// parent thread 4943private static size_t _receiveAsyncLines(Terminator, Unit) 4944 (const(ubyte)[] data, ref EncodingScheme encodingScheme, 4945 bool keepTerminator, Terminator terminator, 4946 ref const(ubyte)[] leftOverBytes, ref bool bufferValid, 4947 ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer, 4948 Tid fromTid, ref bool aborted) 4949{ 4950 import std.format : format; 4951 4952 immutable datalen = data.length; 4953 4954 // Terminator is specified and buffers should be resized as determined by 4955 // the terminator 4956 4957 // Copy data to active buffer until terminator is found. 4958 4959 // Decode as many lines as possible 4960 while (true) 4961 { 4962 4963 // Make sure a buffer is present 4964 while (!bufferValid && freeBuffers.empty) 4965 { 4966 // Active buffer is invalid and there are no available buffers in 4967 // the pool. Wait for buffers to return from main thread in order to 4968 // reuse them. 4969 receive((immutable(Unit)[] buf) 4970 { 4971 buffer = cast(Unit[]) buf; 4972 buffer.length = 0; 4973 buffer.assumeSafeAppend(); 4974 bufferValid = true; 4975 }, 4976 (bool flag) { aborted = true; } 4977 ); 4978 if (aborted) return cast(size_t) 0; 4979 } 4980 if (!bufferValid) 4981 { 4982 buffer = freeBuffers.pop(); 4983 bufferValid = true; 4984 } 4985 4986 // Try to read a line from left over bytes from last onReceive plus the 4987 // newly received bytes. 4988 try 4989 { 4990 if (decodeLineInto(leftOverBytes, data, buffer, 4991 encodingScheme, terminator)) 4992 { 4993 if (keepTerminator) 4994 { 4995 fromTid.send(thisTid, 4996 curlMessage(cast(immutable(Unit)[])buffer)); 4997 } 4998 else 4999 { 5000 static if (isArray!Terminator) 5001 fromTid.send(thisTid, 5002 curlMessage(cast(immutable(Unit)[]) 5003 buffer[0..$-terminator.length])); 5004 else 5005 fromTid.send(thisTid, 5006 curlMessage(cast(immutable(Unit)[]) 5007 buffer[0..$-1])); 5008 } 5009 bufferValid = false; 5010 } 5011 else 5012 { 5013 // Could not decode an entire line. Save 5014 // bytes left in data for next call to 5015 // onReceive. Can be up to a max of 4 bytes. 5016 enforce!CurlException(data.length <= 4, 5017 format( 5018 "Too many bytes left not decoded %s"~ 5019 " > 4. Maybe the charset specified in"~ 5020 " headers does not match "~ 5021 "the actual content downloaded?", 5022 data.length)); 5023 leftOverBytes ~= data; 5024 break; 5025 } 5026 } 5027 catch (CurlException ex) 5028 { 5029 prioritySend(fromTid, cast(immutable(CurlException))ex); 5030 return cast(size_t) 0; 5031 } 5032 } 5033 return datalen; 5034} 5035 5036// ditto 5037private static 5038void _finalizeAsyncLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid) 5039{ 5040 if (bufferValid && buffer.length != 0) 5041 fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$])); 5042} 5043 5044 5045// Spawn a thread for handling the reading of incoming data in the 5046// background while the delegate is executing. This will optimize 5047// throughput by allowing simultaneous input (this struct) and 5048// output (e.g. AsyncHTTPLineOutputRange). 5049private static void _spawnAsync(Conn, Unit, Terminator = void)() 5050{ 5051 Tid fromTid = receiveOnly!Tid(); 5052 5053 // Get buffer to read into 5054 Pool!(Unit[]) freeBuffers; // Free list of buffer objects 5055 5056 // Number of bytes filled into active buffer 5057 Unit[] buffer; 5058 bool aborted = false; 5059 5060 EncodingScheme encodingScheme; 5061 static if ( !is(Terminator == void)) 5062 { 5063 // Only lines reading will receive a terminator 5064 const terminator = receiveOnly!Terminator(); 5065 const keepTerminator = receiveOnly!bool(); 5066 5067 // max number of bytes to carry over from an onReceive 5068 // callback. This is 4 because it is the max code units to 5069 // decode a code point in the supported encodings. 5070 auto leftOverBytes = new const(ubyte)[4]; 5071 leftOverBytes.length = 0; 5072 auto bufferValid = false; 5073 } 5074 else 5075 { 5076 Unit[] outdata; 5077 } 5078 5079 // no move semantic available in std.concurrency ie. must use casting. 5080 auto connDup = cast(CURL*) receiveOnly!ulong(); 5081 auto client = Conn(); 5082 client.p.curl.handle = connDup; 5083 5084 // receive a method for both ftp and http but just use it for http 5085 auto method = receiveOnly!(HTTP.Method)(); 5086 5087 client.onReceive = (ubyte[] data) 5088 { 5089 // If no terminator is specified the chunk size is fixed. 5090 static if ( is(Terminator == void) ) 5091 return _receiveAsyncChunks(data, outdata, freeBuffers, buffer, 5092 fromTid, aborted); 5093 else 5094 return _receiveAsyncLines(data, encodingScheme, 5095 keepTerminator, terminator, leftOverBytes, 5096 bufferValid, freeBuffers, buffer, 5097 fromTid, aborted); 5098 }; 5099 5100 static if ( is(Conn == HTTP) ) 5101 { 5102 client.method = method; 5103 // register dummy header handler 5104 client.onReceiveHeader = (in char[] key, in char[] value) 5105 { 5106 if (key == "content-type") 5107 encodingScheme = EncodingScheme.create(client.p.charset); 5108 }; 5109 } 5110 else 5111 { 5112 encodingScheme = EncodingScheme.create(client.encoding); 5113 } 5114 5115 // Start the request 5116 CurlCode code; 5117 try 5118 { 5119 code = client.perform(No.throwOnError); 5120 } 5121 catch (Exception ex) 5122 { 5123 prioritySend(fromTid, cast(immutable(Exception)) ex); 5124 fromTid.send(thisTid, curlMessage(true)); // signal done 5125 return; 5126 } 5127 5128 if (code != CurlError.ok) 5129 { 5130 if (aborted && (code == CurlError.aborted_by_callback || 5131 code == CurlError.write_error)) 5132 { 5133 fromTid.send(thisTid, curlMessage(true)); // signal done 5134 return; 5135 } 5136 prioritySend(fromTid, cast(immutable(CurlException)) 5137 new CurlException(client.p.curl.errorString(code))); 5138 5139 fromTid.send(thisTid, curlMessage(true)); // signal done 5140 return; 5141 } 5142 5143 // Send remaining data that is not a full chunk size 5144 static if ( is(Terminator == void) ) 5145 _finalizeAsyncChunks(outdata, buffer, fromTid); 5146 else 5147 _finalizeAsyncLines(bufferValid, buffer, fromTid); 5148 5149 fromTid.send(thisTid, curlMessage(true)); // signal done 5150} 5151