uri.d revision 1.1.1.1 
1// Written in the D programming language.
2
3/**
4 * Encode and decode Uniform Resource Identifiers (URIs).
5 * URIs are used in internet transfer protocols.
6 * Valid URI characters consist of letters, digits,
7 * and the characters $(B ;/?:@&=+$,-_.!~*'())
8 * Reserved URI characters are $(B ;/?:@&=+$,)
9 * Escape sequences consist of $(B %) followed by two hex digits.
10 *
11 * See_Also:
12 *  $(LINK2 http://www.ietf.org/rfc/rfc3986.txt, RFC 3986)<br>
13 *  $(LINK2 http://en.wikipedia.org/wiki/Uniform_resource_identifier, Wikipedia)
14 * Copyright: Copyright Digital Mars 2000 - 2009.
15 * License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
16 * Authors:   $(HTTP digitalmars.com, Walter Bright)
17 * Source:    $(PHOBOSSRC std/_uri.d)
18 */
19/*          Copyright Digital Mars 2000 - 2009.
20 * Distributed under the Boost Software License, Version 1.0.
21 *    (See accompanying file LICENSE_1_0.txt or copy at
22 *          http://www.boost.org/LICENSE_1_0.txt)
23 */
24module std.uri;
25
26//debug=uri;        // uncomment to turn on debugging writefln's
27debug(uri) import std.stdio;
28import std.traits : isSomeChar;
29
30/** This Exception is thrown if something goes wrong when encoding or
31decoding a URI.
32*/
33class URIException : Exception
34{
35    import std.exception : basicExceptionCtors;
36    mixin basicExceptionCtors;
37}
38
39private enum
40{
41    URI_Alpha = 1,
42    URI_Reserved = 2,
43    URI_Mark = 4,
44    URI_Digit = 8,
45    URI_Hash = 0x10,        // '#'
46}
47
48private immutable char[16] hex2ascii = "0123456789ABCDEF";
49
50private immutable ubyte[128] uri_flags =      // indexed by character
51    ({
52        ubyte[128] uflags;
53
54        // Compile time initialize
55        uflags['#'] |= URI_Hash;
56
57        foreach (c; 'A' .. 'Z' + 1)
58        {
59            uflags[c] |= URI_Alpha;
60            uflags[c + 0x20] |= URI_Alpha;   // lowercase letters
61        }
62        foreach (c; '0' .. '9' + 1) uflags[c] |= URI_Digit;
63        foreach (c; ";/?:@&=+$,")   uflags[c] |= URI_Reserved;
64        foreach (c; "-_.!~*'()")    uflags[c] |= URI_Mark;
65        return uflags;
66    })();
67
68private string URI_Encode(dstring str, uint unescapedSet)
69{
70    import core.exception : OutOfMemoryError;
71    import core.stdc.stdlib : alloca;
72
73    uint j;
74    uint k;
75    dchar V;
76    dchar C;
77
78    // result buffer
79    char[50] buffer = void;
80    char* R;
81    uint Rlen;
82    uint Rsize; // alloc'd size
83
84    immutable len = str.length;
85
86    R = buffer.ptr;
87    Rsize = buffer.length;
88    Rlen = 0;
89
90    for (k = 0; k != len; k++)
91    {
92        C = str[k];
93        // if (C in unescapedSet)
94        if (C < uri_flags.length && uri_flags[C] & unescapedSet)
95        {
96            if (Rlen == Rsize)
97            {
98                char* R2;
99
100                Rsize *= 2;
101                if (Rsize > 1024)
102                {
103                    R2 = (new char[Rsize]).ptr;
104                }
105                else
106                {
107                    R2 = cast(char *) alloca(Rsize * char.sizeof);
108                    if (!R2)
109                        throw new OutOfMemoryError("Alloca failure");
110                }
111                R2[0 .. Rlen] = R[0 .. Rlen];
112                R = R2;
113            }
114            R[Rlen] = cast(char) C;
115            Rlen++;
116        }
117        else
118        {
119            char[6] Octet;
120            uint L;
121
122            V = C;
123
124            // Transform V into octets
125            if (V <= 0x7F)
126            {
127                Octet[0] = cast(char) V;
128                L = 1;
129            }
130            else if (V <= 0x7FF)
131            {
132                Octet[0] = cast(char)(0xC0 | (V >> 6));
133                Octet[1] = cast(char)(0x80 | (V & 0x3F));
134                L = 2;
135            }
136            else if (V <= 0xFFFF)
137            {
138                Octet[0] = cast(char)(0xE0 | (V >> 12));
139                Octet[1] = cast(char)(0x80 | ((V >> 6) & 0x3F));
140                Octet[2] = cast(char)(0x80 | (V & 0x3F));
141                L = 3;
142            }
143            else if (V <= 0x1FFFFF)
144            {
145                Octet[0] = cast(char)(0xF0 | (V >> 18));
146                Octet[1] = cast(char)(0x80 | ((V >> 12) & 0x3F));
147                Octet[2] = cast(char)(0x80 | ((V >> 6) & 0x3F));
148                Octet[3] = cast(char)(0x80 | (V & 0x3F));
149                L = 4;
150            }
151            else
152            {
153                throw new URIException("Undefined UTF-32 code point");
154            }
155
156            if (Rlen + L * 3 > Rsize)
157            {
158                char *R2;
159
160                Rsize = 2 * (Rlen + L * 3);
161                if (Rsize > 1024)
162                {
163                    R2 = (new char[Rsize]).ptr;
164                }
165                else
166                {
167                    R2 = cast(char *) alloca(Rsize * char.sizeof);
168                    if (!R2)
169                        throw new OutOfMemoryError("Alloca failure");
170                }
171                R2[0 .. Rlen] = R[0 .. Rlen];
172                R = R2;
173            }
174
175            for (j = 0; j < L; j++)
176            {
177                R[Rlen] = '%';
178                R[Rlen + 1] = hex2ascii[Octet[j] >> 4];
179                R[Rlen + 2] = hex2ascii[Octet[j] & 15];
180
181                Rlen += 3;
182            }
183        }
184    }
185
186    return R[0 .. Rlen].idup;
187}
188
189private uint ascii2hex(dchar c) @nogc @safe pure nothrow
190{
191    return (c <= '9') ? c - '0' :
192        (c <= 'F') ? c - 'A' + 10 :
193        c - 'a' + 10;
194}
195
196private dstring URI_Decode(Char)(in Char[] uri, uint reservedSet)
197if (isSomeChar!Char)
198{
199    import core.exception : OutOfMemoryError;
200    import core.stdc.stdlib : alloca;
201    import std.ascii : isHexDigit;
202
203    uint j;
204    uint k;
205    uint V;
206    dchar C;
207
208    // Result array, allocated on stack
209    dchar* R;
210    uint Rlen;
211
212    immutable len = uri.length;
213    auto s = uri.ptr;
214
215    // Preallocate result buffer R guaranteed to be large enough for result
216    auto Rsize = len;
217    if (Rsize > 1024 / dchar.sizeof)
218    {
219        R = (new dchar[Rsize]).ptr;
220    }
221    else
222    {
223        R = cast(dchar *) alloca(Rsize * dchar.sizeof);
224        if (!R)
225            throw new OutOfMemoryError("Alloca failure");
226    }
227    Rlen = 0;
228
229    for (k = 0; k != len; k++)
230    {
231        char B;
232        uint start;
233
234        C = s[k];
235        if (C != '%')
236        {
237            R[Rlen] = C;
238            Rlen++;
239            continue;
240        }
241        start = k;
242        if (k + 2 >= len)
243            throw new URIException("Unexpected end of URI");
244        if (!isHexDigit(s[k + 1]) || !isHexDigit(s[k + 2]))
245            throw new URIException("Expected two hexadecimal digits after '%'");
246        B = cast(char)((ascii2hex(s[k + 1]) << 4) + ascii2hex(s[k + 2]));
247        k += 2;
248        if ((B & 0x80) == 0)
249        {
250            C = B;
251        }
252        else
253        {
254            uint n;
255
256            for (n = 1; ; n++)
257            {
258                if (n > 4)
259                    throw new URIException("UTF-32 code point size too large");
260                if (((B << n) & 0x80) == 0)
261                {
262                    if (n == 1)
263                        throw new URIException("UTF-32 code point size too small");
264                    break;
265                }
266            }
267
268            // Pick off (7 - n) significant bits of B from first byte of octet
269            V = B & ((1 << (7 - n)) - 1);   // (!!!)
270
271            if (k + (3 * (n - 1)) >= len)
272                throw new URIException("UTF-32 unaligned String");
273            for (j = 1; j != n; j++)
274            {
275                k++;
276                if (s[k] != '%')
277                    throw new URIException("Expected: '%'");
278                if (!isHexDigit(s[k + 1]) || !isHexDigit(s[k + 2]))
279                    throw new URIException("Expected two hexadecimal digits after '%'");
280                B = cast(char)((ascii2hex(s[k + 1]) << 4) + ascii2hex(s[k + 2]));
281                if ((B & 0xC0) != 0x80)
282                    throw new URIException("Incorrect UTF-32 multi-byte sequence");
283                k += 2;
284                V = (V << 6) | (B & 0x3F);
285            }
286            if (V > 0x10FFFF)
287                throw new URIException("Unknown UTF-32 code point");
288            C = V;
289        }
290        if (C < uri_flags.length && uri_flags[C] & reservedSet)
291        {
292            // R ~= s[start .. k + 1];
293            immutable width = (k + 1) - start;
294            for (int ii = 0; ii < width; ii++)
295                R[Rlen + ii] = s[start + ii];
296            Rlen += width;
297        }
298        else
299        {
300            R[Rlen] = C;
301            Rlen++;
302        }
303    }
304    assert(Rlen <= Rsize);  // enforce our preallocation size guarantee
305
306    // Copy array on stack to array in memory
307    return R[0 .. Rlen].idup;
308}
309
310/*************************************
311 * Decodes the URI string encodedURI into a UTF-8 string and returns it.
312 * Escape sequences that resolve to reserved URI characters are not replaced.
313 * Escape sequences that resolve to the '#' character are not replaced.
314 */
315
316string decode(Char)(in Char[] encodedURI)
317if (isSomeChar!Char)
318{
319    import std.algorithm.iteration : each;
320    import std.utf : encode;
321    auto s = URI_Decode(encodedURI, URI_Reserved | URI_Hash);
322    char[] r;
323    s.each!(c => encode(r, c));
324    return r;
325}
326
327/*******************************
328 * Decodes the URI string encodedURI into a UTF-8 string and returns it. All
329 * escape sequences are decoded.
330 */
331
332string decodeComponent(Char)(in Char[] encodedURIComponent)
333if (isSomeChar!Char)
334{
335    import std.algorithm.iteration : each;
336    import std.utf : encode;
337    auto s = URI_Decode(encodedURIComponent, 0);
338    char[] r;
339    s.each!(c => encode(r, c));
340    return r;
341}
342
343/*****************************
344 * Encodes the UTF-8 string uri into a URI and returns that URI. Any character
345 * not a valid URI character is escaped. The '#' character is not escaped.
346 */
347
348string encode(Char)(in Char[] uri)
349if (isSomeChar!Char)
350{
351    import std.utf : toUTF32;
352    auto s = toUTF32(uri);
353    return URI_Encode(s, URI_Reserved | URI_Hash | URI_Alpha | URI_Digit | URI_Mark);
354}
355
356/********************************
357 * Encodes the UTF-8 string uriComponent into a URI and returns that URI.
358 * Any character not a letter, digit, or one of -_.!~*'() is escaped.
359 */
360
361string encodeComponent(Char)(in Char[] uriComponent)
362if (isSomeChar!Char)
363{
364    import std.utf : toUTF32;
365    auto s = toUTF32(uriComponent);
366    return URI_Encode(s, URI_Alpha | URI_Digit | URI_Mark);
367}
368
369/* Encode associative array using www-form-urlencoding
370 *
371 * Params:
372 *      values = an associative array containing the values to be encoded.
373 *
374 * Returns:
375 *      A string encoded using www-form-urlencoding.
376 */
377package string urlEncode(in string[string] values)
378{
379    if (values.length == 0)
380        return "";
381
382    import std.array : Appender;
383    import std.format : formattedWrite;
384
385    Appender!string enc;
386    enc.reserve(values.length * 128);
387
388    bool first = true;
389    foreach (k, v; values)
390    {
391        if (!first)
392            enc.put('&');
393        formattedWrite(enc, "%s=%s", encodeComponent(k), encodeComponent(v));
394        first = false;
395    }
396    return enc.data;
397}
398
399@system unittest
400{
401    // @system because urlEncode -> encodeComponent -> URI_Encode
402    // URI_Encode uses alloca and pointer slicing
403    string[string] a;
404    assert(urlEncode(a) == "");
405    assert(urlEncode(["name1" : "value1"]) == "name1=value1");
406    auto enc = urlEncode(["name1" : "value1", "name2" : "value2"]);
407    assert(enc == "name1=value1&name2=value2" || enc == "name2=value2&name1=value1");
408}
409
410/***************************
411 * Does string s[] start with a URL?
412 * Returns:
413 *  -1   it does not
414 *  len  it does, and s[0 .. len] is the slice of s[] that is that URL
415 */
416
417ptrdiff_t uriLength(Char)(in Char[] s)
418if (isSomeChar!Char)
419{
420    /* Must start with one of:
421     *  http://
422     *  https://
423     *  www.
424     */
425    import std.ascii : isAlphaNum;
426    import std.uni : icmp;
427
428    ptrdiff_t i;
429
430    if (s.length <= 4)
431        return -1;
432
433    if (s.length > 7 && icmp(s[0 .. 7], "http://") == 0)
434    {
435        i = 7;
436    }
437    else
438    {
439        if (s.length > 8 && icmp(s[0 .. 8], "https://") == 0)
440            i = 8;
441        else
442            return -1;
443    }
444
445    ptrdiff_t lastdot;
446    for (; i < s.length; i++)
447    {
448        auto c = s[i];
449        if (isAlphaNum(c))
450            continue;
451        if (c == '-' || c == '_' || c == '?' ||
452                c == '=' || c == '%' || c == '&' ||
453                c == '/' || c == '+' || c == '#' ||
454                c == '~' || c == '$')
455            continue;
456        if (c == '.')
457        {
458            lastdot = i;
459            continue;
460        }
461        break;
462    }
463    if (!lastdot)
464        return -1;
465
466    return i;
467}
468
469///
470@safe unittest
471{
472    string s1 = "http://www.digitalmars.com/~fred/fredsRX.html#foo end!";
473    assert(uriLength(s1) == 49);
474    string s2 = "no uri here";
475    assert(uriLength(s2) == -1);
476    assert(uriLength("issue 14924") < 0);
477}
478
479
480/***************************
481 * Does string s[] start with an email address?
482 * Returns:
483 *  -1    it does not
484 *  len   it does, and s[0 .. i] is the slice of s[] that is that email address
485 * References:
486 *  RFC2822
487 */
488ptrdiff_t emailLength(Char)(in Char[] s)
489if (isSomeChar!Char)
490{
491    import std.ascii : isAlpha, isAlphaNum;
492
493    ptrdiff_t i;
494
495    if (!isAlpha(s[0]))
496        return -1;
497
498    for (i = 1; 1; i++)
499    {
500        if (i == s.length)
501            return -1;
502        auto c = s[i];
503        if (isAlphaNum(c))
504            continue;
505        if (c == '-' || c == '_' || c == '.')
506            continue;
507        if (c != '@')
508            return -1;
509        i++;
510        break;
511    }
512
513    /* Now do the part past the '@'
514     */
515    ptrdiff_t lastdot;
516    for (; i < s.length; i++)
517    {
518        auto c = s[i];
519        if (isAlphaNum(c))
520            continue;
521        if (c == '-' || c == '_')
522            continue;
523        if (c == '.')
524        {
525            lastdot = i;
526            continue;
527        }
528        break;
529    }
530    if (!lastdot || (i - lastdot != 3 && i - lastdot != 4))
531        return -1;
532
533    return i;
534}
535
536///
537@safe unittest
538{
539    string s1 = "my.e-mail@www.example-domain.com with garbage added";
540    assert(emailLength(s1) == 32);
541    string s2 = "no email address here";
542    assert(emailLength(s2) == -1);
543    assert(emailLength("issue 14924") < 0);
544}
545
546
547@system unittest
548{
549    //@system because of encode -> URI_Encode
550    debug(uri) writeln("uri.encodeURI.unittest");
551
552    string source = "http://www.digitalmars.com/~fred/fred's RX.html#foo";
553    string target = "http://www.digitalmars.com/~fred/fred's%20RX.html#foo";
554
555    auto result = encode(source);
556    debug(uri) writefln("result = '%s'", result);
557    assert(result == target);
558    result = decode(target);
559    debug(uri) writefln("result = '%s'", result);
560    assert(result == source);
561
562    result = encode(decode("%E3%81%82%E3%81%82"));
563    assert(result == "%E3%81%82%E3%81%82");
564
565    result = encodeComponent("c++");
566    assert(result == "c%2B%2B");
567
568    auto str = new char[10_000_000];
569    str[] = 'A';
570    result = encodeComponent(str);
571    foreach (char c; result)
572        assert(c == 'A');
573
574    result = decode("%41%42%43");
575    debug(uri) writeln(result);
576
577    import std.meta : AliasSeq;
578    foreach (StringType; AliasSeq!(char[], wchar[], dchar[], string, wstring, dstring))
579    {
580        import std.conv : to;
581        StringType decoded1 = source.to!StringType;
582        string encoded1 = encode(decoded1);
583        assert(decoded1 == source.to!StringType); // check that `decoded1` wasn't changed
584        assert(encoded1 == target);
585        assert(decoded1 == decode(encoded1).to!StringType);
586
587        StringType encoded2 = target.to!StringType;
588        string decoded2 = decode(encoded2);
589        assert(encoded2 == target.to!StringType); // check that `encoded2` wasn't changed
590        assert(decoded2 == source);
591        assert(encoded2 == encode(decoded2).to!StringType);
592    }
593}
594