encodedstream.h

00001 // Tencent is pleased to support the open source community by making RapidJSON available.
00002 // 
00003 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
00004 //
00005 // Licensed under the MIT License (the "License"); you may not use this file except
00006 // in compliance with the License. You may obtain a copy of the License at
00007 //
00008 // http://opensource.org/licenses/MIT
00009 //
00010 // Unless required by applicable law or agreed to in writing, software distributed 
00011 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
00012 // CONDITIONS OF ANY KIND, either express or implied. See the License for the 
00013 // specific language governing permissions and limitations under the License.
00014 
00015 #ifndef RAPIDJSON_ENCODEDSTREAM_H_
00016 #define RAPIDJSON_ENCODEDSTREAM_H_
00017 
00018 #include "stream.h"
00019 #include "memorystream.h"
00020 
00021 #ifdef __GNUC__
00022 RAPIDJSON_DIAG_PUSH
00023 RAPIDJSON_DIAG_OFF(effc++)
00024 #endif
00025 
00026 #ifdef __clang__
00027 RAPIDJSON_DIAG_PUSH
00028 RAPIDJSON_DIAG_OFF(padded)
00029 #endif
00030 
00031 RAPIDJSON_NAMESPACE_BEGIN
00032 
00033 //! Input byte stream wrapper with a statically bound encoding.
00034 /*!
00035     \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE.
00036     \tparam InputByteStream Type of input byte stream. For example, FileReadStream.
00037 */
00038 template <typename Encoding, typename InputByteStream>
00039 class EncodedInputStream {
00040     RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
00041 public:
00042     typedef typename Encoding::Ch Ch;
00043 
00044     EncodedInputStream(InputByteStream& is) : is_(is) { 
00045         current_ = Encoding::TakeBOM(is_);
00046     }
00047 
00048     Ch Peek() const { return current_; }
00049     Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; }
00050     size_t Tell() const { return is_.Tell(); }
00051 
00052     // Not implemented
00053     void Put(Ch) { RAPIDJSON_ASSERT(false); }
00054     void Flush() { RAPIDJSON_ASSERT(false); } 
00055     Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
00056     size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
00057 
00058 private:
00059     EncodedInputStream(const EncodedInputStream&);
00060     EncodedInputStream& operator=(const EncodedInputStream&);
00061 
00062     InputByteStream& is_;
00063     Ch current_;
00064 };
00065 
00066 //! Specialized for UTF8 MemoryStream.
00067 template <>
00068 class EncodedInputStream<UTF8<>, MemoryStream> {
00069 public:
00070     typedef UTF8<>::Ch Ch;
00071 
00072     EncodedInputStream(MemoryStream& is) : is_(is) {
00073         if (static_cast<unsigned char>(is_.Peek()) == 0xEFu) is_.Take();
00074         if (static_cast<unsigned char>(is_.Peek()) == 0xBBu) is_.Take();
00075         if (static_cast<unsigned char>(is_.Peek()) == 0xBFu) is_.Take();
00076     }
00077     Ch Peek() const { return is_.Peek(); }
00078     Ch Take() { return is_.Take(); }
00079     size_t Tell() const { return is_.Tell(); }
00080 
00081     // Not implemented
00082     void Put(Ch) {}
00083     void Flush() {} 
00084     Ch* PutBegin() { return 0; }
00085     size_t PutEnd(Ch*) { return 0; }
00086 
00087     MemoryStream& is_;
00088 
00089 private:
00090     EncodedInputStream(const EncodedInputStream&);
00091     EncodedInputStream& operator=(const EncodedInputStream&);
00092 };
00093 
00094 //! Output byte stream wrapper with statically bound encoding.
00095 /*!
00096     \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE.
00097     \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream.
00098 */
00099 template <typename Encoding, typename OutputByteStream>
00100 class EncodedOutputStream {
00101     RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
00102 public:
00103     typedef typename Encoding::Ch Ch;
00104 
00105     EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { 
00106         if (putBOM)
00107             Encoding::PutBOM(os_);
00108     }
00109 
00110     void Put(Ch c) { Encoding::Put(os_, c);  }
00111     void Flush() { os_.Flush(); }
00112 
00113     // Not implemented
00114     Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;}
00115     Ch Take() { RAPIDJSON_ASSERT(false); return 0;}
00116     size_t Tell() const { RAPIDJSON_ASSERT(false);  return 0; }
00117     Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
00118     size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
00119 
00120 private:
00121     EncodedOutputStream(const EncodedOutputStream&);
00122     EncodedOutputStream& operator=(const EncodedOutputStream&);
00123 
00124     OutputByteStream& os_;
00125 };
00126 
00127 #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x
00128 
00129 //! Input stream wrapper with dynamically bound encoding and automatic encoding detection.
00130 /*!
00131     \tparam CharType Type of character for reading.
00132     \tparam InputByteStream type of input byte stream to be wrapped.
00133 */
00134 template <typename CharType, typename InputByteStream>
00135 class AutoUTFInputStream {
00136     RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
00137 public:
00138     typedef CharType Ch;
00139 
00140     //! Constructor.
00141     /*!
00142         \param is input stream to be wrapped.
00143         \param type UTF encoding type if it is not detected from the stream.
00144     */
00145     AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) {
00146         RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE);        
00147         DetectType();
00148         static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) };
00149         takeFunc_ = f[type_];
00150         current_ = takeFunc_(*is_);
00151     }
00152 
00153     UTFType GetType() const { return type_; }
00154     bool HasBOM() const { return hasBOM_; }
00155 
00156     Ch Peek() const { return current_; }
00157     Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; }
00158     size_t Tell() const { return is_->Tell(); }
00159 
00160     // Not implemented
00161     void Put(Ch) { RAPIDJSON_ASSERT(false); }
00162     void Flush() { RAPIDJSON_ASSERT(false); } 
00163     Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
00164     size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
00165 
00166 private:
00167     AutoUTFInputStream(const AutoUTFInputStream&);
00168     AutoUTFInputStream& operator=(const AutoUTFInputStream&);
00169 
00170     // Detect encoding type with BOM or RFC 4627
00171     void DetectType() {
00172         // BOM (Byte Order Mark):
00173         // 00 00 FE FF  UTF-32BE
00174         // FF FE 00 00  UTF-32LE
00175         // FE FF        UTF-16BE
00176         // FF FE        UTF-16LE
00177         // EF BB BF     UTF-8
00178 
00179         const unsigned char* c = reinterpret_cast<const unsigned char *>(is_->Peek4());
00180         if (!c)
00181             return;
00182 
00183         unsigned bom = static_cast<unsigned>(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24));
00184         hasBOM_ = false;
00185         if (bom == 0xFFFE0000)                  { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); }
00186         else if (bom == 0x0000FEFF)             { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); }
00187         else if ((bom & 0xFFFF) == 0xFFFE)      { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take();                           }
00188         else if ((bom & 0xFFFF) == 0xFEFF)      { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take();                           }
00189         else if ((bom & 0xFFFFFF) == 0xBFBBEF)  { type_ = kUTF8;    hasBOM_ = true; is_->Take(); is_->Take(); is_->Take();              }
00190 
00191         // RFC 4627: Section 3
00192         // "Since the first two characters of a JSON text will always be ASCII
00193         // characters [RFC0020], it is possible to determine whether an octet
00194         // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking
00195         // at the pattern of nulls in the first four octets."
00196         // 00 00 00 xx  UTF-32BE
00197         // 00 xx 00 xx  UTF-16BE
00198         // xx 00 00 00  UTF-32LE
00199         // xx 00 xx 00  UTF-16LE
00200         // xx xx xx xx  UTF-8
00201 
00202         if (!hasBOM_) {
00203             unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0);
00204             switch (pattern) {
00205             case 0x08: type_ = kUTF32BE; break;
00206             case 0x0A: type_ = kUTF16BE; break;
00207             case 0x01: type_ = kUTF32LE; break;
00208             case 0x05: type_ = kUTF16LE; break;
00209             case 0x0F: type_ = kUTF8;    break;
00210             default: break; // Use type defined by user.
00211             }
00212         }
00213 
00214         // Runtime check whether the size of character type is sufficient. It only perform checks with assertion.
00215         if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2);
00216         if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4);
00217     }
00218 
00219     typedef Ch (*TakeFunc)(InputByteStream& is);
00220     InputByteStream* is_;
00221     UTFType type_;
00222     Ch current_;
00223     TakeFunc takeFunc_;
00224     bool hasBOM_;
00225 };
00226 
00227 //! Output stream wrapper with dynamically bound encoding and automatic encoding detection.
00228 /*!
00229     \tparam CharType Type of character for writing.
00230     \tparam OutputByteStream type of output byte stream to be wrapped.
00231 */
00232 template <typename CharType, typename OutputByteStream>
00233 class AutoUTFOutputStream {
00234     RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
00235 public:
00236     typedef CharType Ch;
00237 
00238     //! Constructor.
00239     /*!
00240         \param os output stream to be wrapped.
00241         \param type UTF encoding type.
00242         \param putBOM Whether to write BOM at the beginning of the stream.
00243     */
00244     AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) {
00245         RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE);
00246 
00247         // Runtime check whether the size of character type is sufficient. It only perform checks with assertion.
00248         if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2);
00249         if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4);
00250 
00251         static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) };
00252         putFunc_ = f[type_];
00253 
00254         if (putBOM)
00255             PutBOM();
00256     }
00257 
00258     UTFType GetType() const { return type_; }
00259 
00260     void Put(Ch c) { putFunc_(*os_, c); }
00261     void Flush() { os_->Flush(); } 
00262 
00263     // Not implemented
00264     Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;}
00265     Ch Take() { RAPIDJSON_ASSERT(false); return 0;}
00266     size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; }
00267     Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
00268     size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
00269 
00270 private:
00271     AutoUTFOutputStream(const AutoUTFOutputStream&);
00272     AutoUTFOutputStream& operator=(const AutoUTFOutputStream&);
00273 
00274     void PutBOM() { 
00275         typedef void (*PutBOMFunc)(OutputByteStream&);
00276         static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) };
00277         f[type_](*os_);
00278     }
00279 
00280     typedef void (*PutFunc)(OutputByteStream&, Ch);
00281 
00282     OutputByteStream* os_;
00283     UTFType type_;
00284     PutFunc putFunc_;
00285 };
00286 
00287 #undef RAPIDJSON_ENCODINGS_FUNC
00288 
00289 RAPIDJSON_NAMESPACE_END
00290 
00291 #ifdef __clang__
00292 RAPIDJSON_DIAG_POP
00293 #endif
00294 
00295 #ifdef __GNUC__
00296 RAPIDJSON_DIAG_POP
00297 #endif
00298 
00299 #endif // RAPIDJSON_FILESTREAM_H_