1/*
2 * Copyright 2010-2021 Haiku Inc. All rights reserved.
3 * Distributed under the terms of the MIT License.
4 *
5 * Authors:
6 *		Christophe Huriaux, c.huriaux@gmail.com
7 *		Niels Sascha Reedijk, niels.reedijk@gmail.com
8 *		Adrien Destugues, pulkomandy@pulkomandy.tk
9 *		Stephan A��mus, superstippi@gmx.de
10 */
11
12
13#include <HttpRequest.h>
14
15#include <arpa/inet.h>
16#include <stdio.h>
17
18#include <cstdlib>
19#include <deque>
20#include <new>
21
22#include <AutoDeleter.h>
23#include <Certificate.h>
24#include <Debug.h>
25#include <DynamicBuffer.h>
26#include <File.h>
27#include <ProxySecureSocket.h>
28#include <Socket.h>
29#include <SecureSocket.h>
30#include <StackOrHeapArray.h>
31#include <ZlibCompressionAlgorithm.h>
32
33using namespace BPrivate::Network;
34
35
36static const int32 kHttpBufferSize = 4096;
37
38
39namespace BPrivate {
40
41	class CheckedSecureSocket: public BSecureSocket
42	{
43		public:
44			CheckedSecureSocket(BHttpRequest* request);
45
46			bool			CertificateVerificationFailed(BCertificate& certificate,
47					const char* message);
48
49		private:
50			BHttpRequest*	fRequest;
51	};
52
53
54	CheckedSecureSocket::CheckedSecureSocket(BHttpRequest* request)
55		:
56		BSecureSocket(),
57		fRequest(request)
58	{
59	}
60
61
62	bool
63	CheckedSecureSocket::CertificateVerificationFailed(BCertificate& certificate,
64		const char* message)
65	{
66		return fRequest->_CertificateVerificationFailed(certificate, message);
67	}
68
69
70	class CheckedProxySecureSocket: public BProxySecureSocket
71	{
72		public:
73			CheckedProxySecureSocket(const BNetworkAddress& proxy, BHttpRequest* request);
74
75			bool			CertificateVerificationFailed(BCertificate& certificate,
76					const char* message);
77
78		private:
79			BHttpRequest*	fRequest;
80	};
81
82
83	CheckedProxySecureSocket::CheckedProxySecureSocket(const BNetworkAddress& proxy,
84		BHttpRequest* request)
85		:
86		BProxySecureSocket(proxy),
87		fRequest(request)
88	{
89	}
90
91
92	bool
93	CheckedProxySecureSocket::CertificateVerificationFailed(BCertificate& certificate,
94		const char* message)
95	{
96		return fRequest->_CertificateVerificationFailed(certificate, message);
97	}
98};
99
100
101BHttpRequest::BHttpRequest(const BUrl& url, BDataIO* output, bool ssl,
102	const char* protocolName, BUrlProtocolListener* listener,
103	BUrlContext* context)
104	:
105	BNetworkRequest(url, output, listener, context, "BUrlProtocol.HTTP",
106		protocolName),
107	fSSL(ssl),
108	fRequestMethod(B_HTTP_GET),
109	fHttpVersion(B_HTTP_11),
110	fResult(url),
111	fRequestStatus(kRequestInitialState),
112	fOptHeaders(NULL),
113	fOptPostFields(NULL),
114	fOptInputData(NULL),
115	fOptInputDataSize(-1),
116	fOptRangeStart(-1),
117	fOptRangeEnd(-1),
118	fOptFollowLocation(true)
119{
120	_ResetOptions();
121	fSocket = NULL;
122}
123
124
125BHttpRequest::BHttpRequest(const BHttpRequest& other)
126	:
127	BNetworkRequest(other.Url(), other.Output(), other.fListener,
128		other.fContext, "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
129	fSSL(other.fSSL),
130	fRequestMethod(other.fRequestMethod),
131	fHttpVersion(other.fHttpVersion),
132	fResult(other.fUrl),
133	fRequestStatus(kRequestInitialState),
134	fOptHeaders(NULL),
135	fOptPostFields(NULL),
136	fOptInputData(NULL),
137	fOptInputDataSize(-1),
138	fOptRangeStart(other.fOptRangeStart),
139	fOptRangeEnd(other.fOptRangeEnd),
140	fOptFollowLocation(other.fOptFollowLocation)
141{
142	_ResetOptions();
143		// FIXME some options may be copied from other instead.
144	fSocket = NULL;
145}
146
147
148BHttpRequest::~BHttpRequest()
149{
150	Stop();
151
152	delete fSocket;
153
154	delete fOptInputData;
155	delete fOptHeaders;
156	delete fOptPostFields;
157}
158
159
160void
161BHttpRequest::SetMethod(const char* const method)
162{
163	fRequestMethod = method;
164}
165
166
167void
168BHttpRequest::SetFollowLocation(bool follow)
169{
170	fOptFollowLocation = follow;
171}
172
173
174void
175BHttpRequest::SetMaxRedirections(int8 redirections)
176{
177	fOptMaxRedirs = redirections;
178}
179
180
181void
182BHttpRequest::SetReferrer(const BString& referrer)
183{
184	fOptReferer = referrer;
185}
186
187
188void
189BHttpRequest::SetUserAgent(const BString& agent)
190{
191	fOptUserAgent = agent;
192}
193
194
195void
196BHttpRequest::SetDiscardData(bool discard)
197{
198	fOptDiscardData = discard;
199}
200
201
202void
203BHttpRequest::SetDisableListener(bool disable)
204{
205	fOptDisableListener = disable;
206}
207
208
209void
210BHttpRequest::SetAutoReferrer(bool enable)
211{
212	fOptAutoReferer = enable;
213}
214
215
216void
217BHttpRequest::SetStopOnError(bool stop)
218{
219	fOptStopOnError = stop;
220}
221
222
223void
224BHttpRequest::SetUserName(const BString& name)
225{
226	fOptUsername = name;
227}
228
229
230void
231BHttpRequest::SetPassword(const BString& password)
232{
233	fOptPassword = password;
234}
235
236
237void
238BHttpRequest::SetRangeStart(off_t position)
239{
240	// This field is used within the transfer loop, so only
241	// allow setting it before sending the request.
242	if (fRequestStatus == kRequestInitialState)
243		fOptRangeStart = position;
244}
245
246
247void
248BHttpRequest::SetRangeEnd(off_t position)
249{
250	// This field could be used in the transfer loop, so only
251	// allow setting it before sending the request.
252	if (fRequestStatus == kRequestInitialState)
253		fOptRangeEnd = position;
254}
255
256
257void
258BHttpRequest::SetPostFields(const BHttpForm& fields)
259{
260	AdoptPostFields(new(std::nothrow) BHttpForm(fields));
261}
262
263
264void
265BHttpRequest::SetHeaders(const BHttpHeaders& headers)
266{
267	AdoptHeaders(new(std::nothrow) BHttpHeaders(headers));
268}
269
270
271void
272BHttpRequest::AdoptPostFields(BHttpForm* const fields)
273{
274	delete fOptPostFields;
275	fOptPostFields = fields;
276
277	if (fOptPostFields != NULL)
278		fRequestMethod = B_HTTP_POST;
279}
280
281
282void
283BHttpRequest::AdoptInputData(BDataIO* const data, const ssize_t size)
284{
285	delete fOptInputData;
286	fOptInputData = data;
287	fOptInputDataSize = size;
288}
289
290
291void
292BHttpRequest::AdoptHeaders(BHttpHeaders* const headers)
293{
294	delete fOptHeaders;
295	fOptHeaders = headers;
296}
297
298
299/*static*/ bool
300BHttpRequest::IsInformationalStatusCode(int16 code)
301{
302	return (code >= B_HTTP_STATUS__INFORMATIONAL_BASE)
303		&& (code <  B_HTTP_STATUS__INFORMATIONAL_END);
304}
305
306
307/*static*/ bool
308BHttpRequest::IsSuccessStatusCode(int16 code)
309{
310	return (code >= B_HTTP_STATUS__SUCCESS_BASE)
311		&& (code <  B_HTTP_STATUS__SUCCESS_END);
312}
313
314
315/*static*/ bool
316BHttpRequest::IsRedirectionStatusCode(int16 code)
317{
318	return (code >= B_HTTP_STATUS__REDIRECTION_BASE)
319		&& (code <  B_HTTP_STATUS__REDIRECTION_END);
320}
321
322
323/*static*/ bool
324BHttpRequest::IsClientErrorStatusCode(int16 code)
325{
326	return (code >= B_HTTP_STATUS__CLIENT_ERROR_BASE)
327		&& (code <  B_HTTP_STATUS__CLIENT_ERROR_END);
328}
329
330
331/*static*/ bool
332BHttpRequest::IsServerErrorStatusCode(int16 code)
333{
334	return (code >= B_HTTP_STATUS__SERVER_ERROR_BASE)
335		&& (code <  B_HTTP_STATUS__SERVER_ERROR_END);
336}
337
338
339/*static*/ int16
340BHttpRequest::StatusCodeClass(int16 code)
341{
342	if (BHttpRequest::IsInformationalStatusCode(code))
343		return B_HTTP_STATUS_CLASS_INFORMATIONAL;
344	else if (BHttpRequest::IsSuccessStatusCode(code))
345		return B_HTTP_STATUS_CLASS_SUCCESS;
346	else if (BHttpRequest::IsRedirectionStatusCode(code))
347		return B_HTTP_STATUS_CLASS_REDIRECTION;
348	else if (BHttpRequest::IsClientErrorStatusCode(code))
349		return B_HTTP_STATUS_CLASS_CLIENT_ERROR;
350	else if (BHttpRequest::IsServerErrorStatusCode(code))
351		return B_HTTP_STATUS_CLASS_SERVER_ERROR;
352
353	return B_HTTP_STATUS_CLASS_INVALID;
354}
355
356
357const BUrlResult&
358BHttpRequest::Result() const
359{
360	return fResult;
361}
362
363
364status_t
365BHttpRequest::Stop()
366{
367
368	if (fSocket != NULL) {
369		fSocket->Disconnect();
370			// Unlock any pending connect, read or write operation.
371	}
372	return BNetworkRequest::Stop();
373}
374
375
376void
377BHttpRequest::_ResetOptions()
378{
379	delete fOptPostFields;
380	delete fOptHeaders;
381
382	fOptFollowLocation = true;
383	fOptMaxRedirs = 8;
384	fOptReferer = "";
385	fOptUserAgent = "Services Kit (Haiku)";
386	fOptUsername = "";
387	fOptPassword = "";
388	fOptAuthMethods = B_HTTP_AUTHENTICATION_BASIC | B_HTTP_AUTHENTICATION_DIGEST
389		| B_HTTP_AUTHENTICATION_IE_DIGEST;
390	fOptHeaders = NULL;
391	fOptPostFields = NULL;
392	fOptSetCookies = true;
393	fOptDiscardData = false;
394	fOptDisableListener = false;
395	fOptAutoReferer = true;
396}
397
398
399status_t
400BHttpRequest::_ProtocolLoop()
401{
402	// Initialize the request redirection loop
403	int8 maxRedirs = fOptMaxRedirs;
404	bool newRequest;
405
406	do {
407		newRequest = false;
408
409		// Result reset
410		fHeaders.Clear();
411		_ResultHeaders().Clear();
412
413		BString host = fUrl.Host();
414		int port = fSSL ? 443 : 80;
415
416		if (fUrl.HasPort())
417			port = fUrl.Port();
418
419		if (fContext->UseProxy()) {
420			host = fContext->GetProxyHost();
421			port = fContext->GetProxyPort();
422		}
423
424		status_t result = fInputBuffer.InitCheck();
425		if (result != B_OK)
426			return result;
427
428		if (!_ResolveHostName(host, port)) {
429			_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR,
430				"Unable to resolve hostname (%s), aborting.",
431					fUrl.Host().String());
432			return B_SERVER_NOT_FOUND;
433		}
434
435		status_t requestStatus = _MakeRequest();
436		if (requestStatus != B_OK)
437			return requestStatus;
438
439		// Prepare the referer for the next request if needed
440		if (fOptAutoReferer)
441			fOptReferer = fUrl.UrlString();
442
443		switch (StatusCodeClass(fResult.StatusCode())) {
444			case B_HTTP_STATUS_CLASS_INFORMATIONAL:
445				// Header 100:continue should have been
446				// handled in the _MakeRequest read loop
447				break;
448
449			case B_HTTP_STATUS_CLASS_SUCCESS:
450				break;
451
452			case B_HTTP_STATUS_CLASS_REDIRECTION:
453			{
454				// Redirection has been explicitly disabled
455				if (!fOptFollowLocation)
456					break;
457
458				int code = fResult.StatusCode();
459				if (code == B_HTTP_STATUS_MOVED_PERMANENTLY
460					|| code == B_HTTP_STATUS_FOUND
461					|| code == B_HTTP_STATUS_SEE_OTHER
462					|| code == B_HTTP_STATUS_TEMPORARY_REDIRECT) {
463					BString locationUrl = fHeaders["Location"];
464
465					fUrl = BUrl(fUrl, locationUrl);
466
467					// 302 and 303 redirections also convert POST requests to GET
468					// (and remove the posted form data)
469					if ((code == B_HTTP_STATUS_FOUND
470						|| code == B_HTTP_STATUS_SEE_OTHER)
471						&& fRequestMethod == B_HTTP_POST) {
472						SetMethod(B_HTTP_GET);
473						delete fOptPostFields;
474						fOptPostFields = NULL;
475						delete fOptInputData;
476						fOptInputData = NULL;
477						fOptInputDataSize = 0;
478					}
479
480					if (--maxRedirs > 0) {
481						newRequest = true;
482
483						// Redirections may need a switch from http to https.
484						if (fUrl.Protocol() == "https")
485							fSSL = true;
486						else if (fUrl.Protocol() == "http")
487							fSSL = false;
488
489						_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
490							"Following: %s\n",
491							fUrl.UrlString().String());
492					}
493				}
494				break;
495			}
496
497			case B_HTTP_STATUS_CLASS_CLIENT_ERROR:
498				if (fResult.StatusCode() == B_HTTP_STATUS_UNAUTHORIZED) {
499					BHttpAuthentication* authentication
500						= &fContext->GetAuthentication(fUrl);
501					status_t status = B_OK;
502
503					if (authentication->Method() == B_HTTP_AUTHENTICATION_NONE) {
504						// There is no authentication context for this
505						// url yet, so let's create one.
506						BHttpAuthentication newAuth;
507						newAuth.Initialize(fHeaders["WWW-Authenticate"]);
508						fContext->AddAuthentication(fUrl, newAuth);
509
510						// Get the copy of the authentication we just added.
511						// That copy is owned by the BUrlContext and won't be
512						// deleted (unlike the temporary object above)
513						authentication = &fContext->GetAuthentication(fUrl);
514					}
515
516					newRequest = false;
517					if (fOptUsername.Length() > 0 && status == B_OK) {
518						// If we received an username and password, add them
519						// to the request. This will either change the
520						// credentials for an existing request, or set them
521						// for a new one we created just above.
522						//
523						// If this request handles HTTP redirections, it will
524						// also automatically retry connecting and send the
525						// login information.
526						authentication->SetUserName(fOptUsername);
527						authentication->SetPassword(fOptPassword);
528						newRequest = true;
529					}
530				}
531				break;
532
533			case B_HTTP_STATUS_CLASS_SERVER_ERROR:
534				break;
535
536			default:
537			case B_HTTP_STATUS_CLASS_INVALID:
538				break;
539		}
540	} while (newRequest);
541
542	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
543		"%" B_PRId32 " headers and %" B_PRIuSIZE " bytes of data remaining",
544		fHeaders.CountHeaders(), fInputBuffer.Size());
545
546	if (fResult.StatusCode() == 404)
547		return B_RESOURCE_NOT_FOUND;
548
549	return B_OK;
550}
551
552
553status_t
554BHttpRequest::_MakeRequest()
555{
556	delete fSocket;
557
558	if (fSSL) {
559		if (fContext->UseProxy()) {
560			BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort());
561			fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this);
562		} else
563			fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this);
564	} else
565		fSocket = new(std::nothrow) BSocket();
566
567	if (fSocket == NULL)
568		return B_NO_MEMORY;
569
570	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
571		fUrl.Authority().String(), fRemoteAddr.Port());
572	status_t connectError = fSocket->Connect(fRemoteAddr);
573
574	if (connectError != B_OK) {
575		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
576			strerror(connectError));
577		return connectError;
578	}
579
580	//! ProtocolHook:ConnectionOpened
581	if (fListener != NULL)
582		fListener->ConnectionOpened(this);
583
584	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
585		"Connection opened, sending request.");
586
587	BString requestHeaders;
588	requestHeaders.Append(_SerializeRequest());
589	requestHeaders.Append(_SerializeHeaders());
590	requestHeaders.Append("\r\n");
591	fSocket->Write(requestHeaders.String(), requestHeaders.Length());
592	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");
593
594	_SendPostData();
595	fRequestStatus = kRequestInitialState;
596
597
598
599	// Receive loop
600	bool disableListener = false;
601	bool receiveEnd = false;
602	bool parseEnd = false;
603	bool readByChunks = false;
604	bool decompress = false;
605	status_t readError = B_OK;
606	ssize_t bytesRead = 0;
607	off_t bytesReceived = 0;
608	off_t bytesTotal = 0;
609	size_t previousBufferSize = 0;
610	off_t bytesUnpacked = 0;
611	char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
612	ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer);
613	ssize_t inputTempSize = kHttpBufferSize;
614	ssize_t chunkSize = -1;
615	DynamicBuffer decompressorStorage;
616	BDataIO* decompressingStream;
617	ObjectDeleter<BDataIO> decompressingStreamDeleter;
618
619	while (!fQuit && !(receiveEnd && parseEnd)) {
620		if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) {
621			BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
622			bytesRead = fSocket->Read(chunk, kHttpBufferSize);
623
624			if (bytesRead < 0) {
625				readError = bytesRead;
626				break;
627			} else if (bytesRead == 0) {
628				// Check if we got the expected number of bytes.
629				// Exceptions:
630				// - If the content-length is not known (bytesTotal is 0), for
631				//   example in the case of a chunked transfer, we can't know
632				// - If the request method is "HEAD" which explicitly asks the
633				//   server to not send any data (only the headers)
634				if (bytesTotal > 0 && bytesReceived != bytesTotal) {
635					readError = B_IO_ERROR;
636					break;
637				}
638				receiveEnd = true;
639			}
640
641			fInputBuffer.AppendData(chunk, bytesRead);
642		} else
643			bytesRead = 0;
644
645		previousBufferSize = fInputBuffer.Size();
646
647		if (fRequestStatus < kRequestStatusReceived) {
648			_ParseStatus();
649
650			if (fOptFollowLocation
651					&& IsRedirectionStatusCode(fResult.StatusCode()))
652				disableListener = true;
653
654			if (fOptStopOnError
655					&& fResult.StatusCode() >= B_HTTP_STATUS_CLASS_CLIENT_ERROR)
656			{
657				fQuit = true;
658				break;
659			}
660
661			//! ProtocolHook:ResponseStarted
662			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
663					&& !disableListener)
664				fListener->ResponseStarted(this);
665		}
666
667		if (fRequestStatus < kRequestHeadersReceived) {
668			_ParseHeaders();
669
670			if (fRequestStatus >= kRequestHeadersReceived) {
671				_ResultHeaders() = fHeaders;
672
673				// Parse received cookies
674				if (fContext != NULL) {
675					for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
676						if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
677							fContext->GetCookieJar().AddCookie(
678								fHeaders.HeaderAt(i).Value(), fUrl);
679						}
680					}
681				}
682
683				//! ProtocolHook:HeadersReceived
684				if (fListener != NULL && !disableListener)
685					fListener->HeadersReceived(this);
686
687
688				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
689					readByChunks = true;
690
691				BString contentEncoding(fHeaders["Content-Encoding"]);
692				// We don't advertise "deflate" support (see above), but we
693				// still try to decompress it, if a server ever sends a deflate
694				// stream despite it not being in our Accept-Encoding list.
695				if (contentEncoding == "gzip"
696						|| contentEncoding == "deflate") {
697					decompress = true;
698					readError = BZlibCompressionAlgorithm()
699						.CreateDecompressingOutputStream(&decompressorStorage,
700							NULL, decompressingStream);
701					if (readError != B_OK)
702						break;
703
704					decompressingStreamDeleter.SetTo(decompressingStream);
705				}
706
707				int32 index = fHeaders.HasHeader("Content-Length");
708				if (index != B_ERROR)
709					bytesTotal = atoll(fHeaders.HeaderAt(index).Value());
710				else
711					bytesTotal = -1;
712
713				if (fRequestMethod == B_HTTP_HEAD
714					|| fResult.StatusCode() == 204) {
715					// In the case of a HEAD request or if the server replies
716					// 204 ("no content"), we don't expect to receive anything
717					// more, and the socket will be closed.
718					receiveEnd = true;
719				}
720			}
721		}
722
723		if (fRequestStatus >= kRequestHeadersReceived) {
724			// If Transfer-Encoding is chunked, we should read a complete
725			// chunk in buffer before handling it
726			if (readByChunks) {
727				if (chunkSize >= 0) {
728					if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
729							// 2 more bytes to handle the closing CR+LF
730						bytesRead = chunkSize;
731						if (inputTempSize < chunkSize + 2) {
732							inputTempSize = chunkSize + 2;
733							inputTempBuffer
734								= new(std::nothrow) char[inputTempSize];
735							inputTempBufferDeleter.SetTo(inputTempBuffer);
736						}
737
738						if (inputTempBuffer == NULL) {
739							readError = B_NO_MEMORY;
740							break;
741						}
742
743						fInputBuffer.RemoveData(inputTempBuffer,
744							chunkSize + 2);
745						chunkSize = -1;
746					} else {
747						// Not enough data, try again later
748						bytesRead = -1;
749					}
750				} else {
751					BString chunkHeader;
752					if (_GetLine(chunkHeader) == B_ERROR) {
753						chunkSize = -1;
754						bytesRead = -1;
755					} else {
756						// Format of a chunk header:
757						// <chunk size in hex>[; optional data]
758						int32 semiColonIndex = chunkHeader.FindFirst(';', 0);
759
760						// Cut-off optional data if present
761						if (semiColonIndex != -1) {
762							chunkHeader.Remove(semiColonIndex,
763								chunkHeader.Length() - semiColonIndex);
764						}
765
766						chunkSize = strtol(chunkHeader.String(), NULL, 16);
767						if (chunkSize == 0)
768							fRequestStatus = kRequestContentReceived;
769
770						bytesRead = -1;
771					}
772				}
773
774				// A chunk of 0 bytes indicates the end of the chunked transfer
775				if (bytesRead == 0)
776					receiveEnd = true;
777			} else {
778				bytesRead = fInputBuffer.Size();
779
780				if (bytesRead > 0) {
781					if (inputTempSize < bytesRead) {
782						inputTempSize = bytesRead;
783						inputTempBuffer = new(std::nothrow) char[bytesRead];
784						inputTempBufferDeleter.SetTo(inputTempBuffer);
785					}
786
787					if (inputTempBuffer == NULL) {
788						readError = B_NO_MEMORY;
789						break;
790					}
791					fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
792				}
793			}
794
795			if (bytesRead >= 0) {
796				bytesReceived += bytesRead;
797
798				if (fOutput != NULL && !disableListener) {
799					if (decompress) {
800						readError = decompressingStream->WriteExactly(
801							inputTempBuffer, bytesRead);
802						if (readError != B_OK)
803							break;
804
805						ssize_t size = decompressorStorage.Size();
806						BStackOrHeapArray<char, 4096> buffer(size);
807						size = decompressorStorage.Read(buffer, size);
808						if (size > 0) {
809							size_t written = 0;
810							readError = fOutput->WriteExactly(buffer,
811								size, &written);
812							if (fListener != NULL && written > 0)
813								fListener->BytesWritten(this, written);
814							if (readError != B_OK)
815								break;
816							bytesUnpacked += size;
817						}
818					} else if (bytesRead > 0) {
819						size_t written = 0;
820						readError = fOutput->WriteExactly(inputTempBuffer,
821							bytesRead, &written);
822						if (fListener != NULL && written > 0)
823							fListener->BytesWritten(this, written);
824						if (readError != B_OK)
825							break;
826					}
827				}
828
829				if (fListener != NULL && !disableListener)
830					fListener->DownloadProgress(this, bytesReceived,
831						std::max((off_t)0, bytesTotal));
832
833				if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
834					receiveEnd = true;
835
836				if (decompress && receiveEnd && !disableListener) {
837					readError = decompressingStream->Flush();
838
839					if (readError == B_BUFFER_OVERFLOW)
840						readError = B_OK;
841
842					if (readError != B_OK)
843						break;
844
845					ssize_t size = decompressorStorage.Size();
846					BStackOrHeapArray<char, 4096> buffer(size);
847					size = decompressorStorage.Read(buffer, size);
848					if (fOutput != NULL && size > 0 && !disableListener) {
849						size_t written = 0;
850						readError = fOutput->WriteExactly(buffer, size,
851							&written);
852						if (fListener != NULL && written > 0)
853							fListener->BytesWritten(this, written);
854						if (readError != B_OK)
855							break;
856						bytesUnpacked += size;
857					}
858				}
859			}
860		}
861
862		parseEnd = (fInputBuffer.Size() == 0);
863	}
864
865	fSocket->Disconnect();
866
867	if (readError != B_OK)
868		return readError;
869
870	return fQuit ? B_INTERRUPTED : B_OK;
871}
872
873
874void
875BHttpRequest::_ParseStatus()
876{
877	// Status line should be formatted like: HTTP/M.m SSS ...
878	// With:   M = Major version of the protocol
879	//         m = Minor version of the protocol
880	//       SSS = three-digit status code of the response
881	//       ... = additional text info
882	BString statusLine;
883	if (_GetLine(statusLine) == B_ERROR)
884		return;
885
886	if (statusLine.CountChars() < 12)
887		return;
888
889	fRequestStatus = kRequestStatusReceived;
890
891	BString statusCodeStr;
892	BString statusText;
893	statusLine.CopyInto(statusCodeStr, 9, 3);
894	_SetResultStatusCode(atoi(statusCodeStr.String()));
895
896	statusLine.CopyInto(_ResultStatusText(), 13, statusLine.Length() - 13);
897
898	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Status line received: Code %d (%s)",
899		atoi(statusCodeStr.String()), _ResultStatusText().String());
900}
901
902
903void
904BHttpRequest::_ParseHeaders()
905{
906	BString currentHeader;
907	while (_GetLine(currentHeader) != B_ERROR) {
908		// An empty line means the end of the header section
909		if (currentHeader.Length() == 0) {
910			fRequestStatus = kRequestHeadersReceived;
911			return;
912		}
913
914		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_IN, "%s",
915			currentHeader.String());
916		fHeaders.AddHeader(currentHeader.String());
917	}
918}
919
920
921BString
922BHttpRequest::_SerializeRequest()
923{
924	BString request(fRequestMethod);
925	request << ' ';
926
927	if (fContext->UseProxy()) {
928		// When there is a proxy, the request must include the host and port so
929		// the proxy knows where to send the request.
930		request << Url().Protocol() << "://" << Url().Host();
931		if (Url().HasPort())
932			request << ':' << Url().Port();
933	}
934
935	if (Url().HasPath() && Url().Path().Length() > 0)
936		request << Url().Path();
937	else
938		request << '/';
939
940	if (Url().HasRequest())
941		request << '?' << Url().Request();
942
943	switch (fHttpVersion) {
944		case B_HTTP_11:
945			request << " HTTP/1.1\r\n";
946			break;
947
948		default:
949		case B_HTTP_10:
950			request << " HTTP/1.0\r\n";
951			break;
952	}
953
954	_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", request.String());
955
956	return request;
957}
958
959
960BString
961BHttpRequest::_SerializeHeaders()
962{
963	BHttpHeaders outputHeaders;
964
965	// HTTP 1.1 additional headers
966	if (fHttpVersion == B_HTTP_11) {
967		BString host = Url().Host();
968		if (Url().HasPort() && !_IsDefaultPort())
969			host << ':' << Url().Port();
970
971		outputHeaders.AddHeader("Host", host);
972
973		outputHeaders.AddHeader("Accept", "*/*");
974		outputHeaders.AddHeader("Accept-Encoding", "gzip");
975			// Allows the server to compress data using the "gzip" format.
976			// "deflate" is not supported, because there are two interpretations
977			// of what it means (the RFC and Microsoft products), and we don't
978			// want to handle this. Very few websites support only deflate,
979			// and most of them will send gzip, or at worst, uncompressed data.
980
981		outputHeaders.AddHeader("Connection", "close");
982			// Let the remote server close the connection after response since
983			// we don't handle multiple request on a single connection
984	}
985
986	// Classic HTTP headers
987	if (fOptUserAgent.CountChars() > 0)
988		outputHeaders.AddHeader("User-Agent", fOptUserAgent.String());
989
990	if (fOptReferer.CountChars() > 0)
991		outputHeaders.AddHeader("Referer", fOptReferer.String());
992
993	// Optional range requests headers
994	if (fOptRangeStart != -1 || fOptRangeEnd != -1) {
995		if (fOptRangeStart == -1)
996			fOptRangeStart = 0;
997		BString range;
998		if (fOptRangeEnd != -1) {
999			range.SetToFormat("bytes=%" B_PRIdOFF "-%" B_PRIdOFF,
1000				fOptRangeStart, fOptRangeEnd);
1001		} else {
1002			range.SetToFormat("bytes=%" B_PRIdOFF "-", fOptRangeStart);
1003		}
1004		outputHeaders.AddHeader("Range", range.String());
1005	}
1006
1007	// Authentication
1008	if (fContext != NULL) {
1009		BHttpAuthentication& authentication = fContext->GetAuthentication(fUrl);
1010		if (authentication.Method() != B_HTTP_AUTHENTICATION_NONE) {
1011			if (fOptUsername.Length() > 0) {
1012				authentication.SetUserName(fOptUsername);
1013				authentication.SetPassword(fOptPassword);
1014			}
1015
1016			BString request(fRequestMethod);
1017			outputHeaders.AddHeader("Authorization",
1018				authentication.Authorization(fUrl, request));
1019		}
1020	}
1021
1022	// Required headers for POST data
1023	if (fOptPostFields != NULL && fRequestMethod == B_HTTP_POST) {
1024		BString contentType;
1025
1026		switch (fOptPostFields->GetFormType()) {
1027			case B_HTTP_FORM_MULTIPART:
1028				contentType << "multipart/form-data; boundary="
1029					<< fOptPostFields->GetMultipartBoundary() << "";
1030				break;
1031
1032			case B_HTTP_FORM_URL_ENCODED:
1033				contentType << "application/x-www-form-urlencoded";
1034				break;
1035		}
1036
1037		outputHeaders.AddHeader("Content-Type", contentType);
1038		outputHeaders.AddHeader("Content-Length",
1039			fOptPostFields->ContentLength());
1040	} else if (fOptInputData != NULL
1041			&& (fRequestMethod == B_HTTP_POST
1042			|| fRequestMethod == B_HTTP_PUT)) {
1043		if (fOptInputDataSize >= 0)
1044			outputHeaders.AddHeader("Content-Length", fOptInputDataSize);
1045		else
1046			outputHeaders.AddHeader("Transfer-Encoding", "chunked");
1047	}
1048
1049	// Optional headers specified by the user
1050	if (fOptHeaders != NULL) {
1051		for (int32 headerIndex = 0; headerIndex < fOptHeaders->CountHeaders();
1052				headerIndex++) {
1053			BHttpHeader& optHeader = (*fOptHeaders)[headerIndex];
1054			int32 replaceIndex = outputHeaders.HasHeader(optHeader.Name());
1055
1056			// Add or replace the current option header to the
1057			// output header list
1058			if (replaceIndex == -1)
1059				outputHeaders.AddHeader(optHeader.Name(), optHeader.Value());
1060			else
1061				outputHeaders[replaceIndex].SetValue(optHeader.Value());
1062		}
1063	}
1064
1065	// Context cookies
1066	if (fOptSetCookies && fContext != NULL) {
1067		BString cookieString;
1068
1069		BNetworkCookieJar::UrlIterator iterator
1070			= fContext->GetCookieJar().GetUrlIterator(fUrl);
1071		const BNetworkCookie* cookie = iterator.Next();
1072		if (cookie != NULL) {
1073			while (true) {
1074				cookieString << cookie->RawCookie(false);
1075				cookie = iterator.Next();
1076				if (cookie == NULL)
1077					break;
1078				cookieString << "; ";
1079			}
1080
1081			outputHeaders.AddHeader("Cookie", cookieString);
1082		}
1083	}
1084
1085	// Write output headers to output stream
1086	BString headerData;
1087
1088	for (int32 headerIndex = 0; headerIndex < outputHeaders.CountHeaders();
1089			headerIndex++) {
1090		const char* header = outputHeaders.HeaderAt(headerIndex).Header();
1091
1092		headerData << header;
1093		headerData << "\r\n";
1094
1095		_EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", header);
1096	}
1097
1098	return headerData;
1099}
1100
1101
1102void
1103BHttpRequest::_SendPostData()
1104{
1105	if (fRequestMethod == B_HTTP_POST && fOptPostFields != NULL) {
1106		if (fOptPostFields->GetFormType() != B_HTTP_FORM_MULTIPART) {
1107			BString outputBuffer = fOptPostFields->RawData();
1108			_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
1109				"%s", outputBuffer.String());
1110			fSocket->Write(outputBuffer.String(), outputBuffer.Length());
1111		} else {
1112			for (BHttpForm::Iterator it = fOptPostFields->GetIterator();
1113				const BHttpFormData* currentField = it.Next();
1114				) {
1115				_EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
1116					it.MultipartHeader().String());
1117				fSocket->Write(it.MultipartHeader().String(),
1118					it.MultipartHeader().Length());
1119
1120				switch (currentField->Type()) {
1121					default:
1122					case B_HTTPFORM_UNKNOWN:
1123						ASSERT(0);
1124						break;
1125
1126					case B_HTTPFORM_STRING:
1127						fSocket->Write(currentField->String().String(),
1128							currentField->String().Length());
1129						break;
1130
1131					case B_HTTPFORM_FILE:
1132						{
1133							BFile upFile(currentField->File().Path(),
1134								B_READ_ONLY);
1135							char readBuffer[kHttpBufferSize];
1136							ssize_t readSize;
1137							off_t totalSize;
1138
1139							if (upFile.GetSize(&totalSize) != B_OK)
1140								ASSERT(0);
1141
1142							readSize = upFile.Read(readBuffer,
1143								sizeof(readBuffer));
1144							while (readSize > 0) {
1145								fSocket->Write(readBuffer, readSize);
1146								readSize = upFile.Read(readBuffer,
1147									sizeof(readBuffer));
1148								fListener->UploadProgress(this, readSize,
1149									std::max((off_t)0, totalSize));
1150							}
1151
1152							break;
1153						}
1154					case B_HTTPFORM_BUFFER:
1155						fSocket->Write(currentField->Buffer(),
1156							currentField->BufferSize());
1157						break;
1158				}
1159
1160				fSocket->Write("\r\n", 2);
1161			}
1162
1163			BString footer = fOptPostFields->GetMultipartFooter();
1164			fSocket->Write(footer.String(), footer.Length());
1165		}
1166	} else if ((fRequestMethod == B_HTTP_POST || fRequestMethod == B_HTTP_PUT)
1167		&& fOptInputData != NULL) {
1168
1169		// If the input data is seekable, we rewind it for each new request.
1170		BPositionIO* seekableData
1171			= dynamic_cast<BPositionIO*>(fOptInputData);
1172		if (seekableData)
1173			seekableData->Seek(0, SEEK_SET);
1174
1175		for (;;) {
1176			char outputTempBuffer[kHttpBufferSize];
1177			ssize_t read = fOptInputData->Read(outputTempBuffer,
1178				sizeof(outputTempBuffer));
1179
1180			if (read <= 0)
1181				break;
1182
1183			if (fOptInputDataSize < 0) {
1184				// Input data size unknown, so we have to use chunked transfer
1185				char hexSize[18];
1186				// The string does not need to be NULL terminated.
1187				size_t hexLength = snprintf(hexSize, sizeof(hexSize), "%lx\r\n",
1188					read);
1189
1190				fSocket->Write(hexSize, hexLength);
1191				fSocket->Write(outputTempBuffer, read);
1192				fSocket->Write("\r\n", 2);
1193			} else {
1194				fSocket->Write(outputTempBuffer, read);
1195			}
1196		}
1197
1198		if (fOptInputDataSize < 0) {
1199			// Chunked transfer terminating sequence
1200			fSocket->Write("0\r\n\r\n", 5);
1201		}
1202	}
1203
1204}
1205
1206
1207BHttpHeaders&
1208BHttpRequest::_ResultHeaders()
1209{
1210	return fResult.fHeaders;
1211}
1212
1213
1214void
1215BHttpRequest::_SetResultStatusCode(int32 statusCode)
1216{
1217	fResult.fStatusCode = statusCode;
1218}
1219
1220
1221BString&
1222BHttpRequest::_ResultStatusText()
1223{
1224	return fResult.fStatusString;
1225}
1226
1227
1228bool
1229BHttpRequest::_CertificateVerificationFailed(BCertificate& certificate,
1230	const char* message)
1231{
1232	if (fContext->HasCertificateException(certificate))
1233		return true;
1234
1235	if (fListener != NULL
1236		&& fListener->CertificateVerificationFailed(this, certificate, message)) {
1237		// User asked us to continue anyway, let's add a temporary exception for this certificate
1238		fContext->AddCertificateException(certificate);
1239		return true;
1240	}
1241
1242	return false;
1243}
1244
1245
1246bool
1247BHttpRequest::_IsDefaultPort()
1248{
1249	if (fSSL && Url().Port() == 443)
1250		return true;
1251	if (!fSSL && Url().Port() == 80)
1252		return true;
1253	return false;
1254}
1255