1 #define __PHYSICSFS_INTERNAL__

     2 #include "physfs_internal.h"

     3 

     4 

     5 /*

     6  * From rfc3629, the UTF-8 spec:

     7  *  http://www.ietf.org/rfc/rfc3629.txt

     8  *

     9  *   Char. number range  |        UTF-8 octet sequence

    10  *      (hexadecimal)    |              (binary)

    11  *   --------------------+---------------------------------------------

    12  *   0000 0000-0000 007F | 0xxxxxxx

    13  *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx

    14  *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx

    15  *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

    16  */

    17 

    18 

    19 /*

    20  * This may not be the best value, but it's one that isn't represented

    21  *  in Unicode (0x10FFFF is the largest codepoint value). We return this

    22  *  value from utf8codepoint() if there's bogus bits in the

    23  *  stream. utf8codepoint() will turn this value into something

    24  *  reasonable (like a question mark), for text that wants to try to recover,

    25  *  whereas utf8valid() will use the value to determine if a string has bad

    26  *  bits.

    27  */

    28 #define UNICODE_BOGUS_CHAR_VALUE 0xFFFFFFFF

    29 

    30 /*

    31  * This is the codepoint we currently return when there was bogus bits in a

    32  *  UTF-8 string. May not fly in Asian locales?

    33  */

    34 #define UNICODE_BOGUS_CHAR_CODEPOINT '?'

    35 

    36 static PHYSFS_uint32 utf8codepoint(const char **_str)

    37 {

    38     const char *str = *_str;

    39     PHYSFS_uint32 retval = 0;

    40     PHYSFS_uint32 octet = (PHYSFS_uint32) ((PHYSFS_uint8) *str);

    41     PHYSFS_uint32 octet2, octet3, octet4;

    42 

    43     if (octet == 0)  /* null terminator, end of string. */

    44         return 0;

    45 

    46     else if (octet < 128)  /* one octet char: 0 to 127 */

    47     {

    48         (*_str)++;  /* skip to next possible start of codepoint. */

    49         return octet;

    50     } /* else if */

    51 

    52     else if ((octet > 127) && (octet < 192))  /* bad (starts with 10xxxxxx). */

    53     {

    54         /*

    55          * Apparently each of these is supposed to be flagged as a bogus

    56          *  char, instead of just resyncing to the next valid codepoint.

    57          */

    58         (*_str)++;  /* skip to next possible start of codepoint. */

    59         return UNICODE_BOGUS_CHAR_VALUE;

    60     } /* else if */

    61 

    62     else if (octet < 224)  /* two octets */

    63     {

    64         (*_str)++;  /* advance at least one byte in case of an error */

    65         octet -= (128+64);

    66         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

    67         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

    68             return UNICODE_BOGUS_CHAR_VALUE;

    69 

    70         *_str += 1;  /* skip to next possible start of codepoint. */

    71         retval = ((octet << 6) | (octet2 - 128));

    72         if ((retval >= 0x80) && (retval <= 0x7FF))

    73             return retval;

    74     } /* else if */

    75 

    76     else if (octet < 240)  /* three octets */

    77     {

    78         (*_str)++;  /* advance at least one byte in case of an error */

    79         octet -= (128+64+32);

    80         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

    81         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

    82             return UNICODE_BOGUS_CHAR_VALUE;

    83 

    84         octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

    85         if ((octet3 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

    86             return UNICODE_BOGUS_CHAR_VALUE;

    87 

    88         *_str += 2;  /* skip to next possible start of codepoint. */

    89         retval = ( ((octet << 12)) | ((octet2-128) << 6) | ((octet3-128)) );

    90 

    91         /* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */

    92         switch (retval)

    93         {

    94             case 0xD800:

    95             case 0xDB7F:

    96             case 0xDB80:

    97             case 0xDBFF:

    98             case 0xDC00:

    99             case 0xDF80:

   100             case 0xDFFF:

   101                 return UNICODE_BOGUS_CHAR_VALUE;

   102         } /* switch */

   103 

   104         /* 0xFFFE and 0xFFFF are illegal, too, so we check them at the edge. */

   105         if ((retval >= 0x800) && (retval <= 0xFFFD))

   106             return retval;

   107     } /* else if */

   108 

   109     else if (octet < 248)  /* four octets */

   110     {

   111         (*_str)++;  /* advance at least one byte in case of an error */

   112         octet -= (128+64+32+16);

   113         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   114         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   115             return UNICODE_BOGUS_CHAR_VALUE;

   116 

   117         octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   118         if ((octet3 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   119             return UNICODE_BOGUS_CHAR_VALUE;

   120 

   121         octet4 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   122         if ((octet4 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   123             return UNICODE_BOGUS_CHAR_VALUE;

   124 

   125         *_str += 3;  /* skip to next possible start of codepoint. */

   126         retval = ( ((octet << 18)) | ((octet2 - 128) << 12) |

   127                    ((octet3 - 128) << 6) | ((octet4 - 128)) );

   128         if ((retval >= 0x10000) && (retval <= 0x10FFFF))

   129             return retval;

   130     } /* else if */

   131 

   132     /*

   133      * Five and six octet sequences became illegal in rfc3629.

   134      *  We throw the codepoint away, but parse them to make sure we move

   135      *  ahead the right number of bytes and don't overflow the buffer.

   136      */

   137 

   138     else if (octet < 252)  /* five octets */

   139     {

   140         (*_str)++;  /* advance at least one byte in case of an error */

   141         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   142         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   143             return UNICODE_BOGUS_CHAR_VALUE;

   144 

   145         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   146         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   147             return UNICODE_BOGUS_CHAR_VALUE;

   148 

   149         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   150         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   151             return UNICODE_BOGUS_CHAR_VALUE;

   152 

   153         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   154         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   155             return UNICODE_BOGUS_CHAR_VALUE;

   156 

   157         *_str += 4;  /* skip to next possible start of codepoint. */

   158         return UNICODE_BOGUS_CHAR_VALUE;

   159     } /* else if */

   160 

   161     else  /* six octets */

   162     {

   163         (*_str)++;  /* advance at least one byte in case of an error */

   164         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   165         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   166             return UNICODE_BOGUS_CHAR_VALUE;

   167 

   168         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   169         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   170             return UNICODE_BOGUS_CHAR_VALUE;

   171 

   172         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   173         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   174             return UNICODE_BOGUS_CHAR_VALUE;

   175 

   176         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   177         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   178             return UNICODE_BOGUS_CHAR_VALUE;

   179 

   180         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));

   181         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */

   182             return UNICODE_BOGUS_CHAR_VALUE;

   183 

   184         *_str += 6;  /* skip to next possible start of codepoint. */

   185         return UNICODE_BOGUS_CHAR_VALUE;

   186     } /* else if */

   187 

   188     return UNICODE_BOGUS_CHAR_VALUE;

   189 } /* utf8codepoint */

   190 

   191 

   192 void PHYSFS_utf8ToUcs4(const char *src, PHYSFS_uint32 *dst, PHYSFS_uint64 len)

   193 {

   194     len -= sizeof (PHYSFS_uint32);   /* save room for null char. */

   195     while (len >= sizeof (PHYSFS_uint32))

   196     {

   197         PHYSFS_uint32 cp = utf8codepoint(&src);

   198         if (cp == 0)

   199             break;

   200         else if (cp == UNICODE_BOGUS_CHAR_VALUE)

   201             cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   202         *(dst++) = cp;

   203         len -= sizeof (PHYSFS_uint32);

   204     } /* while */

   205 

   206     *dst = 0;

   207 } /* PHYSFS_utf8ToUcs4 */

   208 

   209 

   210 void PHYSFS_utf8ToUcs2(const char *src, PHYSFS_uint16 *dst, PHYSFS_uint64 len)

   211 {

   212     len -= sizeof (PHYSFS_uint16);   /* save room for null char. */

   213     while (len >= sizeof (PHYSFS_uint16))

   214     {

   215         PHYSFS_uint32 cp = utf8codepoint(&src);

   216         if (cp == 0)

   217             break;

   218         else if (cp == UNICODE_BOGUS_CHAR_VALUE)

   219             cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   220 

   221         if (cp > 0xFFFF)  /* UTF-16 surrogates (bogus chars in UCS-2) */

   222             cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   223 

   224         *(dst++) = cp;

   225         len -= sizeof (PHYSFS_uint16);

   226     } /* while */

   227 

   228     *dst = 0;

   229 } /* PHYSFS_utf8ToUcs2 */

   230 

   231 

   232 void PHYSFS_utf8ToUtf16(const char *src, PHYSFS_uint16 *dst, PHYSFS_uint64 len)

   233 {

   234     len -= sizeof (PHYSFS_uint16);   /* save room for null char. */

   235     while (len >= sizeof (PHYSFS_uint16))

   236     {

   237         PHYSFS_uint32 cp = utf8codepoint(&src);

   238         if (cp == 0)

   239             break;

   240         else if (cp == UNICODE_BOGUS_CHAR_VALUE)

   241             cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   242 

   243         if (cp > 0xFFFF)  /* encode as surrogate pair */

   244         {

   245             if (len < (sizeof (PHYSFS_uint16) * 2))

   246                 break;  /* not enough room for the pair, stop now. */

   247 

   248             cp -= 0x10000;  /* Make this a 20-bit value */

   249 

   250             *(dst++) = 0xD800 + ((cp >> 10) & 0x3FF);

   251             len -= sizeof (PHYSFS_uint16);

   252 

   253             cp = 0xDC00 + (cp & 0x3FF);

   254         } /* if */

   255 

   256         *(dst++) = cp;

   257         len -= sizeof (PHYSFS_uint16);

   258     } /* while */

   259 

   260     *dst = 0;

   261 } /* PHYSFS_utf8ToUtf16 */

   262 

   263 static void utf8fromcodepoint(PHYSFS_uint32 cp, char **_dst, PHYSFS_uint64 *_len)

   264 {

   265     char *dst = *_dst;

   266     PHYSFS_uint64 len = *_len;

   267 

   268     if (len == 0)

   269         return;

   270 

   271     if (cp > 0x10FFFF)

   272         cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   273     else if ((cp == 0xFFFE) || (cp == 0xFFFF))  /* illegal values. */

   274         cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   275     else

   276     {

   277         /* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */

   278         switch (cp)

   279         {

   280             case 0xD800:

   281             case 0xDB7F:

   282             case 0xDB80:

   283             case 0xDBFF:

   284             case 0xDC00:

   285             case 0xDF80:

   286             case 0xDFFF:

   287                 cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   288         } /* switch */

   289     } /* else */

   290 

   291     /* Do the encoding... */

   292     if (cp < 0x80)

   293     {

   294         *(dst++) = (char) cp;

   295         len--;

   296     } /* if */

   297 

   298     else if (cp < 0x800)

   299     {

   300         if (len < 2)

   301             len = 0;

   302         else

   303         {

   304             *(dst++) = (char) ((cp >> 6) | 128 | 64);

   305             *(dst++) = (char) (cp & 0x3F) | 128;

   306             len -= 2;

   307         } /* else */

   308     } /* else if */

   309 

   310     else if (cp < 0x10000)

   311     {

   312         if (len < 3)

   313             len = 0;

   314         else

   315         {

   316             *(dst++) = (char) ((cp >> 12) | 128 | 64 | 32);

   317             *(dst++) = (char) ((cp >> 6) & 0x3F) | 128;

   318             *(dst++) = (char) (cp & 0x3F) | 128;

   319             len -= 3;

   320         } /* else */

   321     } /* else if */

   322 

   323     else

   324     {

   325         if (len < 4)

   326             len = 0;

   327         else

   328         {

   329             *(dst++) = (char) ((cp >> 18) | 128 | 64 | 32 | 16);

   330             *(dst++) = (char) ((cp >> 12) & 0x3F) | 128;

   331             *(dst++) = (char) ((cp >> 6) & 0x3F) | 128;

   332             *(dst++) = (char) (cp & 0x3F) | 128;

   333             len -= 4;

   334         } /* else if */

   335     } /* else */

   336 

   337     *_dst = dst;

   338     *_len = len;

   339 } /* utf8fromcodepoint */

   340 

   341 #define UTF8FROMTYPE(typ, src, dst, len) \

   342     if (len == 0) return; \

   343     len--;  \

   344     while (len) \

   345     { \

   346         const PHYSFS_uint32 cp = (PHYSFS_uint32) ((typ) (*(src++))); \

   347         if (cp == 0) break; \

   348         utf8fromcodepoint(cp, &dst, &len); \

   349     } \

   350     *dst = '\0'; \

   351 

   352 void PHYSFS_utf8FromUcs4(const PHYSFS_uint32 *src, char *dst, PHYSFS_uint64 len)

   353 {

   354     UTF8FROMTYPE(PHYSFS_uint32, src, dst, len);

   355 } /* PHYSFS_utf8FromUcs4 */

   356 

   357 void PHYSFS_utf8FromUcs2(const PHYSFS_uint16 *src, char *dst, PHYSFS_uint64 len)

   358 {

   359     UTF8FROMTYPE(PHYSFS_uint64, src, dst, len);

   360 } /* PHYSFS_utf8FromUcs2 */

   361 

   362 /* latin1 maps to unicode codepoints directly, we just utf-8 encode it. */

   363 void PHYSFS_utf8FromLatin1(const char *src, char *dst, PHYSFS_uint64 len)

   364 {

   365     UTF8FROMTYPE(PHYSFS_uint8, src, dst, len);

   366 } /* PHYSFS_utf8FromLatin1 */

   367 

   368 #undef UTF8FROMTYPE

   369 

   370 

   371 void PHYSFS_utf8FromUtf16(const PHYSFS_uint16 *src, char *dst, PHYSFS_uint64 len)

   372 {

   373     if (len == 0)

   374         return;

   375 

   376     len--;

   377     while (len)

   378     {

   379         PHYSFS_uint32 cp = (PHYSFS_uint32) *(src++);

   380         if (cp == 0)

   381             break;

   382 

   383         /* Orphaned second half of surrogate pair? */

   384         if ((cp >= 0xDC00) && (cp <= 0xDFFF))

   385             cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   386         else if ((cp >= 0xD800) && (cp <= 0xDBFF))  /* start surrogate pair! */

   387         {

   388             const PHYSFS_uint32 pair = (PHYSFS_uint32) *src;

   389             if ((pair < 0xDC00) || (pair > 0xDFFF))

   390                 cp = UNICODE_BOGUS_CHAR_CODEPOINT;

   391             else

   392             {

   393                 src++;  /* eat the other surrogate. */

   394                 cp = (((cp - 0xD800) << 10) | (pair - 0xDC00));

   395             } /* else */

   396         } /* else if */

   397 

   398         utf8fromcodepoint(cp, &dst, &len);

   399     } /* while */

   400 

   401     *dst = '\0';

   402 } /* PHYSFS_utf8FromUtf16 */

   403 

   404 

   405 typedef struct CaseFoldMapping

   406 {

   407     PHYSFS_uint32 from;

   408     PHYSFS_uint32 to0;

   409     PHYSFS_uint32 to1;

   410     PHYSFS_uint32 to2;

   411 } CaseFoldMapping;

   412 

   413 typedef struct CaseFoldHashBucket

   414 {

   415     const PHYSFS_uint8 count;

   416     const CaseFoldMapping *list;

   417 } CaseFoldHashBucket;

   418 

   419 #include "physfs_casefolding.h"

   420 

   421 static void locate_case_fold_mapping(const PHYSFS_uint32 from,

   422                                      PHYSFS_uint32 *to)

   423 {

   424     PHYSFS_uint32 i;

   425     const PHYSFS_uint8 hashed = ((from ^ (from >> 8)) & 0xFF);

   426     const CaseFoldHashBucket *bucket = &case_fold_hash[hashed];

   427     const CaseFoldMapping *mapping = bucket->list;

   428 

   429     for (i = 0; i < bucket->count; i++, mapping++)

   430     {

   431         if (mapping->from == from)

   432         {

   433             to[0] = mapping->to0;

   434             to[1] = mapping->to1;

   435             to[2] = mapping->to2;

   436             return;

   437         } /* if */

   438     } /* for */

   439 

   440     /* Not found...there's no remapping for this codepoint. */

   441     to[0] = from;

   442     to[1] = 0;

   443     to[2] = 0;

   444 } /* locate_case_fold_mapping */

   445 

   446 

   447 static int utf8codepointcmp(const PHYSFS_uint32 cp1, const PHYSFS_uint32 cp2)

   448 {

   449     PHYSFS_uint32 folded1[3], folded2[3];

   450     locate_case_fold_mapping(cp1, folded1);

   451     locate_case_fold_mapping(cp2, folded2);

   452     return ( (folded1[0] == folded2[0]) &&

   453              (folded1[1] == folded2[1]) &&

   454              (folded1[2] == folded2[2]) );

   455 } /* utf8codepointcmp */

   456 

   457 

   458 int __PHYSFS_utf8stricmp(const char *str1, const char *str2)

   459 {

   460     while (1)

   461     {

   462         const PHYSFS_uint32 cp1 = utf8codepoint(&str1);

   463         const PHYSFS_uint32 cp2 = utf8codepoint(&str2);

   464         if (!utf8codepointcmp(cp1, cp2)) break;

   465         if (cp1 == 0) return 1;

   466     } /* while */

   467 

   468     return 0;

   469 } /* __PHYSFS_utf8stricmp */

   470 

   471 

   472 int __PHYSFS_utf8strnicmp(const char *str1, const char *str2, PHYSFS_uint32 n)

   473 {

   474     while (n > 0)

   475     {

   476         const PHYSFS_uint32 cp1 = utf8codepoint(&str1);

   477         const PHYSFS_uint32 cp2 = utf8codepoint(&str2);

   478         if (!utf8codepointcmp(cp1, cp2)) return 0;

   479         if (cp1 == 0) return 1;

   480         n--;

   481     } /* while */

   482 

   483     return 1;  /* matched to n chars. */

   484 } /* __PHYSFS_utf8strnicmp */

   485 

   486 

   487 int __PHYSFS_stricmpASCII(const char *str1, const char *str2)

   488 {

   489     while (1)

   490     {

   491         const char ch1 = *(str1++);

   492         const char ch2 = *(str2++);

   493         const char cp1 = ((ch1 >= 'A') && (ch1 <= 'Z')) ? (ch1+32) : ch1;

   494         const char cp2 = ((ch2 >= 'A') && (ch2 <= 'Z')) ? (ch2+32) : ch2;

   495         if (cp1 < cp2)

   496             return -1;

   497         else if (cp1 > cp2)

   498             return 1;

   499         else if (cp1 == 0)  /* they're both null chars? */

   500             break;

   501     } /* while */

   502 

   503     return 0;

   504 } /* __PHYSFS_stricmpASCII */

   505 

   506 

   507 int __PHYSFS_strnicmpASCII(const char *str1, const char *str2, PHYSFS_uint32 n)

   508 {

   509     while (n-- > 0)

   510     {

   511         const char ch1 = *(str1++);

   512         const char ch2 = *(str2++);

   513         const char cp1 = ((ch1 >= 'A') && (ch1 <= 'Z')) ? (ch1+32) : ch1;

   514         const char cp2 = ((ch2 >= 'A') && (ch2 <= 'Z')) ? (ch2+32) : ch2;

   515         if (cp1 < cp2)

   516             return -1;

   517         else if (cp1 > cp2)

   518             return 1;

   519         else if (cp1 == 0)  /* they're both null chars? */

   520             return 0;

   521     } /* while */

   522 

   523     return 0;

   524 } /* __PHYSFS_strnicmpASCII */

   525 

   526 

   527 /* end of physfs_unicode.c ... */

   528