vcs.maemo.org Git - someplayer/blob - src/taglib/toolkit/unicode.cpp

   1 /*******************************************************************************
   2  *                                                                             *
   3  * THIS FILE IS INCLUDED IN TAGLIB, BUT IS NOT COPYRIGHTED BY THE TAGLIB       *
   4  * AUTHORS, NOT PART OF THE TAGLIB API AND COULD GO AWAY AT ANY POINT IN TIME. *
   5  * AS SUCH IT SHOULD BE CONSIERED FOR INTERNAL USE ONLY.                       *
   6  *                                                                             *
   7  *******************************************************************************/
   8
   9 /*
  10  * Copyright 2001 Unicode, Inc.
  11  *
  12  * Disclaimer
  13  *
  14  * This source code is provided as is by Unicode, Inc. No claims are
  15  * made as to fitness for any particular purpose. No warranties of any
  16  * kind are expressed or implied. The recipient agrees to determine
  17  * applicability of information provided. If this file has been
  18  * purchased on magnetic or optical media from Unicode, Inc., the
  19  * sole remedy for any claim will be exchange of defective media
  20  * within 90 days of receipt.
  21  *
  22  * Limitations on Rights to Redistribute This Code
  23  *
  24  * Unicode, Inc. hereby grants the right to freely use the information
  25  * supplied in this file in the creation of products supporting the
  26  * Unicode Standard, and to make copies of this file in any form
  27  * for internal or external distribution as long as this notice
  28  * remains attached.
  29  */
  30
  31 /*
  32  * This file has been modified by Scott Wheeler <wheeler@kde.org> to remove
  33  * the UTF32 conversion functions and to place the appropriate functions
  34  * in their own C++ namespace.
  35  */
  36
  37 /* ---------------------------------------------------------------------
  38
  39     Conversions between UTF32, UTF-16, and UTF-8. Source code file.
  40         Author: Mark E. Davis, 1994.
  41         Rev History: Rick McGowan, fixes & updates May 2001.
  42         Sept 2001: fixed const & error conditions per
  43                 mods suggested by S. Parent & A. Lillich.
  44
  45     See the header file "ConvertUTF.h" for complete documentation.
  46
  47 ------------------------------------------------------------------------ */
  48
  49
  50 #include "unicode.h"
  51 #include <stdio.h>
  52
  53 #define UNI_SUR_HIGH_START      (UTF32)0xD800
  54 #define UNI_SUR_HIGH_END        (UTF32)0xDBFF
  55 #define UNI_SUR_LOW_START       (UTF32)0xDC00
  56 #define UNI_SUR_LOW_END         (UTF32)0xDFFF
  57 #define false                   0
  58 #define true                    1
  59
  60 namespace Unicode {
  61
  62 static const int halfShift      = 10; /* used for shifting by 10 bits */
  63
  64 static const UTF32 halfBase     = 0x0010000UL;
  65 static const UTF32 halfMask     = 0x3FFUL;
  66
  67 /*
  68  * Index into the table below with the first byte of a UTF-8 sequence to
  69  * get the number of trailing bytes that are supposed to follow it.
  70  */
  71 static const char trailingBytesForUTF8[256] = {
  72         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  73         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  74         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  75         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  76         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  77         0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  78         1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  79         2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
  80 };
  81
  82 /*
  83  * Magic values subtracted from a buffer value during UTF8 conversion.
  84  * This table contains as many values as there might be trailing bytes
  85  * in a UTF-8 sequence.
  86  */
  87 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
  88                                          0x03C82080UL, 0xFA082080UL, 0x82082080UL };
  89
  90 /*
  91  * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
  92  * into the first byte, depending on how many bytes follow.  There are
  93  * as many entries in this table as there are UTF-8 sequence types.
  94  * (I.e., one byte sequence, two byte... six byte sequence.)
  95  */
  96 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
  97
  98 /* --------------------------------------------------------------------- */
  99
 100 /* The interface converts a whole buffer to avoid function-call overhead.
 101  * Constants have been gathered. Loops & conditionals have been removed as
 102  * much as possible for efficiency, in favor of drop-through switches.
 103  * (See "Note A" at the bottom of the file for equivalent code.)
 104  * If your compiler supports it, the "isLegalUTF8" call can be turned
 105  * into an inline function.
 106  */
 107
 108 /* --------------------------------------------------------------------- */
 109
 110 ConversionResult ConvertUTF16toUTF8 (
 111                 const UTF16** sourceStart, const UTF16* sourceEnd,
 112                 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
 113         ConversionResult result = conversionOK;
 114         const UTF16* source = *sourceStart;
 115         UTF8* target = *targetStart;
 116         while (source < sourceEnd) {
 117                 UTF32 ch;
 118                 unsigned short bytesToWrite = 0;
 119                 const UTF32 byteMask = 0xBF;
 120                 const UTF32 byteMark = 0x80;
 121                 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
 122                 ch = *source++;
 123                 /* If we have a surrogate pair, convert to UTF32 first. */
 124                 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END && source < sourceEnd) {
 125                         UTF32 ch2 = *source;
 126                         if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
 127                                 ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
 128                                         + (ch2 - UNI_SUR_LOW_START) + halfBase;
 129                                 ++source;
 130                         } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
 131                                 --source; /* return to the illegal value itself */
 132                                 result = sourceIllegal;
 133                                 break;
 134                         }
 135                 } else if ((flags == strictConversion) && (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END)) {
 136                         --source; /* return to the illegal value itself */
 137                         result = sourceIllegal;
 138                         break;
 139                 }
 140                 /* Figure out how many bytes the result will require */
 141                 if (ch < (UTF32)0x80) {                 bytesToWrite = 1;
 142                 } else if (ch < (UTF32)0x800) {         bytesToWrite = 2;
 143                 } else if (ch < (UTF32)0x10000) {       bytesToWrite = 3;
 144                 } else if (ch < (UTF32)0x200000) {      bytesToWrite = 4;
 145                 } else {                                bytesToWrite = 2;
 146                                                         ch = UNI_REPLACEMENT_CHAR;
 147                 }
 148                 // printf("bytes to write = %i\n", bytesToWrite);
 149                 target += bytesToWrite;
 150                 if (target > targetEnd) {
 151                         source = oldSource; /* Back up source pointer! */
 152                         target -= bytesToWrite; result = targetExhausted; break;
 153                 }
 154                 switch (bytesToWrite) { /* note: everything falls through. */
 155                         case 4: *--target = (ch | byteMark) & byteMask; ch >>= 6;
 156                         case 3: *--target = (ch | byteMark) & byteMask; ch >>= 6;
 157                         case 2: *--target = (ch | byteMark) & byteMask; ch >>= 6;
 158                         case 1: *--target =  ch | firstByteMark[bytesToWrite];
 159                 }
 160                 target += bytesToWrite;
 161         }
 162         *sourceStart = source;
 163         *targetStart = target;
 164         return result;
 165 }
 166
 167 /* --------------------------------------------------------------------- */
 168
 169 /*
 170  * Utility routine to tell whether a sequence of bytes is legal UTF-8.
 171  * This must be called with the length pre-determined by the first byte.
 172  * If not calling this from ConvertUTF8to*, then the length can be set by:
 173  *      length = trailingBytesForUTF8[*source]+1;
 174  * and the sequence is illegal right away if there aren't that many bytes
 175  * available.
 176  * If presented with a length > 4, this returns false.  The Unicode
 177  * definition of UTF-8 goes up to 4-byte sequences.
 178  */
 179
 180 static Boolean isLegalUTF8(const UTF8 *source, int length) {
 181         UTF8 a;
 182         const UTF8 *srcptr = source+length;
 183         switch (length) {
 184         default: return false;
 185                 /* Everything else falls through when "true"... */
 186         case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
 187         case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
 188         case 2: if ((a = (*--srcptr)) > 0xBF) return false;
 189                 switch (*source) {
 190                     /* no fall-through in this inner switch */
 191                     case 0xE0: if (a < 0xA0) return false; break;
 192                     case 0xF0: if (a < 0x90) return false; break;
 193                     case 0xF4: if (a > 0x8F) return false; break;
 194                     default:  if (a < 0x80) return false;
 195                 }
 196         case 1: if (*source >= 0x80 && *source < 0xC2) return false;
 197                 if (*source > 0xF4) return false;
 198         }
 199         return true;
 200 }
 201
 202 /* --------------------------------------------------------------------- */
 203
 204 /*
 205  * Exported function to return whether a UTF-8 sequence is legal or not.
 206  * This is not used here; it's just exported.
 207  */
 208 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
 209         int length = trailingBytesForUTF8[*source]+1;
 210         if (source+length > sourceEnd) {
 211             return false;
 212         }
 213         return isLegalUTF8(source, length);
 214 }
 215
 216 /* --------------------------------------------------------------------- */
 217
 218 ConversionResult ConvertUTF8toUTF16 (
 219                 const UTF8** sourceStart, const UTF8* sourceEnd,
 220                 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
 221         ConversionResult result = conversionOK;
 222         const UTF8* source = *sourceStart;
 223         UTF16* target = *targetStart;
 224         while (source < sourceEnd) {
 225                 UTF32 ch = 0;
 226                 unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
 227                 if (source + extraBytesToRead >= sourceEnd) {
 228                         result = sourceExhausted; break;
 229                 }
 230                 /* Do this check whether lenient or strict */
 231                 if (! isLegalUTF8(source, extraBytesToRead+1)) {
 232                         result = sourceIllegal;
 233                         break;
 234                 }
 235                 /*
 236                  * The cases all fall through. See "Note A" below.
 237                  */
 238                 switch (extraBytesToRead) {
 239                         case 3: ch += *source++; ch <<= 6;
 240                         case 2: ch += *source++; ch <<= 6;
 241                         case 1: ch += *source++; ch <<= 6;
 242                         case 0: ch += *source++;
 243                 }
 244                 ch -= offsetsFromUTF8[extraBytesToRead];
 245
 246                 if (target >= targetEnd) {
 247                         source -= (extraBytesToRead+1); /* Back up source pointer! */
 248                         result = targetExhausted; break;
 249                 }
 250                 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
 251                         if ((flags == strictConversion) && (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)) {
 252                                 source -= (extraBytesToRead+1); /* return to the illegal value itself */
 253                                 result = sourceIllegal;
 254                                 break;
 255                         } else {
 256                             *target++ = ch;     /* normal case */
 257                         }
 258                 } else if (ch > UNI_MAX_UTF16) {
 259                         if (flags == strictConversion) {
 260                                 result = sourceIllegal;
 261                                 source -= (extraBytesToRead+1); /* return to the start */
 262                                 break; /* Bail out; shouldn't continue */
 263                         } else {
 264                                 *target++ = UNI_REPLACEMENT_CHAR;
 265                         }
 266                 } else {
 267                         /* target is a character in range 0xFFFF - 0x10FFFF. */
 268                         if (target + 1 >= targetEnd) {
 269                                 source -= (extraBytesToRead+1); /* Back up source pointer! */
 270                                 result = targetExhausted; break;
 271                         }
 272                         ch -= halfBase;
 273                         *target++ = (ch >> halfShift) + UNI_SUR_HIGH_START;
 274                         *target++ = (ch & halfMask) + UNI_SUR_LOW_START;
 275                 }
 276         }
 277         *sourceStart = source;
 278         *targetStart = target;
 279         return result;
 280 }
 281
 282 }
 283
 284 /* ---------------------------------------------------------------------
 285
 286         Note A.
 287         The fall-through switches in UTF-8 reading code save a
 288         temp variable, some decrements & conditionals.  The switches
 289         are equivalent to the following loop:
 290                 {
 291                         int tmpBytesToRead = extraBytesToRead+1;
 292                         do {
 293                                 ch += *source++;
 294                                 --tmpBytesToRead;
 295                                 if (tmpBytesToRead) ch <<= 6;
 296                         } while (tmpBytesToRead > 0);
 297                 }
 298         In UTF-8 writing code, the switches on "bytesToWrite" are
 299         similarly unrolled loops.
 300
 301    --------------------------------------------------------------------- */
 302
 303