Colin Hogben / micropython

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Dependents:   micropython-repl

py/unicode.c@0:5868e8752d44, 2016-04-16 (annotated)

Committer:
pythontech
Date:
Sat Apr 16 17:11:56 2016 +0000
Revision:
0:5868e8752d44
Split off library from repl

Who changed what in which revision?

UserRevisionLine numberNew contents of line
pythontech 0:5868e8752d44 1 /*
pythontech 0:5868e8752d44 2 * This file is part of the Micro Python project, http://micropython.org/
pythontech 0:5868e8752d44 3 *
pythontech 0:5868e8752d44 4 * The MIT License (MIT)
pythontech 0:5868e8752d44 5 *
pythontech 0:5868e8752d44 6 * Copyright (c) 2013, 2014 Damien P. George
pythontech 0:5868e8752d44 7 *
pythontech 0:5868e8752d44 8 * Permission is hereby granted, free of charge, to any person obtaining a copy
pythontech 0:5868e8752d44 9 * of this software and associated documentation files (the "Software"), to deal
pythontech 0:5868e8752d44 10 * in the Software without restriction, including without limitation the rights
pythontech 0:5868e8752d44 11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
pythontech 0:5868e8752d44 12 * copies of the Software, and to permit persons to whom the Software is
pythontech 0:5868e8752d44 13 * furnished to do so, subject to the following conditions:
pythontech 0:5868e8752d44 14 *
pythontech 0:5868e8752d44 15 * The above copyright notice and this permission notice shall be included in
pythontech 0:5868e8752d44 16 * all copies or substantial portions of the Software.
pythontech 0:5868e8752d44 17 *
pythontech 0:5868e8752d44 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
pythontech 0:5868e8752d44 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
pythontech 0:5868e8752d44 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
pythontech 0:5868e8752d44 21 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
pythontech 0:5868e8752d44 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
pythontech 0:5868e8752d44 23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
pythontech 0:5868e8752d44 24 * THE SOFTWARE.
pythontech 0:5868e8752d44 25 */
pythontech 0:5868e8752d44 26
pythontech 0:5868e8752d44 27 #include <stdint.h>
pythontech 0:5868e8752d44 28
pythontech 0:5868e8752d44 29 #include "py/unicode.h"
pythontech 0:5868e8752d44 30
pythontech 0:5868e8752d44 31 // attribute flags
pythontech 0:5868e8752d44 32 #define FL_PRINT (0x01)
pythontech 0:5868e8752d44 33 #define FL_SPACE (0x02)
pythontech 0:5868e8752d44 34 #define FL_DIGIT (0x04)
pythontech 0:5868e8752d44 35 #define FL_ALPHA (0x08)
pythontech 0:5868e8752d44 36 #define FL_UPPER (0x10)
pythontech 0:5868e8752d44 37 #define FL_LOWER (0x20)
pythontech 0:5868e8752d44 38 #define FL_XDIGIT (0x40)
pythontech 0:5868e8752d44 39
pythontech 0:5868e8752d44 40 // shorthand character attributes
pythontech 0:5868e8752d44 41 #define AT_PR (FL_PRINT)
pythontech 0:5868e8752d44 42 #define AT_SP (FL_SPACE | FL_PRINT)
pythontech 0:5868e8752d44 43 #define AT_DI (FL_DIGIT | FL_PRINT | FL_XDIGIT)
pythontech 0:5868e8752d44 44 #define AT_AL (FL_ALPHA | FL_PRINT)
pythontech 0:5868e8752d44 45 #define AT_UP (FL_UPPER | FL_ALPHA | FL_PRINT)
pythontech 0:5868e8752d44 46 #define AT_LO (FL_LOWER | FL_ALPHA | FL_PRINT)
pythontech 0:5868e8752d44 47 #define AT_UX (FL_UPPER | FL_ALPHA | FL_PRINT | FL_XDIGIT)
pythontech 0:5868e8752d44 48 #define AT_LX (FL_LOWER | FL_ALPHA | FL_PRINT | FL_XDIGIT)
pythontech 0:5868e8752d44 49
pythontech 0:5868e8752d44 50 // table of attributes for ascii characters
pythontech 0:5868e8752d44 51 STATIC const uint8_t attr[] = {
pythontech 0:5868e8752d44 52 0, 0, 0, 0, 0, 0, 0, 0,
pythontech 0:5868e8752d44 53 0, AT_SP, AT_SP, AT_SP, AT_SP, AT_SP, 0, 0,
pythontech 0:5868e8752d44 54 0, 0, 0, 0, 0, 0, 0, 0,
pythontech 0:5868e8752d44 55 0, 0, 0, 0, 0, 0, 0, 0,
pythontech 0:5868e8752d44 56 AT_SP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
pythontech 0:5868e8752d44 57 AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
pythontech 0:5868e8752d44 58 AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI,
pythontech 0:5868e8752d44 59 AT_DI, AT_DI, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
pythontech 0:5868e8752d44 60 AT_PR, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UP,
pythontech 0:5868e8752d44 61 AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP,
pythontech 0:5868e8752d44 62 AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP,
pythontech 0:5868e8752d44 63 AT_UP, AT_UP, AT_UP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR,
pythontech 0:5868e8752d44 64 AT_PR, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LO,
pythontech 0:5868e8752d44 65 AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO,
pythontech 0:5868e8752d44 66 AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO,
pythontech 0:5868e8752d44 67 AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0
pythontech 0:5868e8752d44 68 };
pythontech 0:5868e8752d44 69
pythontech 0:5868e8752d44 70 // TODO: Rename to str_get_char
pythontech 0:5868e8752d44 71 unichar utf8_get_char(const byte *s) {
pythontech 0:5868e8752d44 72 #if MICROPY_PY_BUILTINS_STR_UNICODE
pythontech 0:5868e8752d44 73 unichar ord = *s++;
pythontech 0:5868e8752d44 74 if (!UTF8_IS_NONASCII(ord)) return ord;
pythontech 0:5868e8752d44 75 ord &= 0x7F;
pythontech 0:5868e8752d44 76 for (unichar mask = 0x40; ord & mask; mask >>= 1) {
pythontech 0:5868e8752d44 77 ord &= ~mask;
pythontech 0:5868e8752d44 78 }
pythontech 0:5868e8752d44 79 while (UTF8_IS_CONT(*s)) {
pythontech 0:5868e8752d44 80 ord = (ord << 6) | (*s++ & 0x3F);
pythontech 0:5868e8752d44 81 }
pythontech 0:5868e8752d44 82 return ord;
pythontech 0:5868e8752d44 83 #else
pythontech 0:5868e8752d44 84 return *s;
pythontech 0:5868e8752d44 85 #endif
pythontech 0:5868e8752d44 86 }
pythontech 0:5868e8752d44 87
pythontech 0:5868e8752d44 88 // TODO: Rename to str_next_char
pythontech 0:5868e8752d44 89 const byte *utf8_next_char(const byte *s) {
pythontech 0:5868e8752d44 90 #if MICROPY_PY_BUILTINS_STR_UNICODE
pythontech 0:5868e8752d44 91 ++s;
pythontech 0:5868e8752d44 92 while (UTF8_IS_CONT(*s)) {
pythontech 0:5868e8752d44 93 ++s;
pythontech 0:5868e8752d44 94 }
pythontech 0:5868e8752d44 95 return s;
pythontech 0:5868e8752d44 96 #else
pythontech 0:5868e8752d44 97 return s + 1;
pythontech 0:5868e8752d44 98 #endif
pythontech 0:5868e8752d44 99 }
pythontech 0:5868e8752d44 100
pythontech 0:5868e8752d44 101 mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) {
pythontech 0:5868e8752d44 102 mp_uint_t i = 0;
pythontech 0:5868e8752d44 103 while (ptr > s) {
pythontech 0:5868e8752d44 104 if (!UTF8_IS_CONT(*--ptr)) {
pythontech 0:5868e8752d44 105 i++;
pythontech 0:5868e8752d44 106 }
pythontech 0:5868e8752d44 107 }
pythontech 0:5868e8752d44 108
pythontech 0:5868e8752d44 109 return i;
pythontech 0:5868e8752d44 110 }
pythontech 0:5868e8752d44 111
pythontech 0:5868e8752d44 112 // TODO: Rename to str_charlen
pythontech 0:5868e8752d44 113 mp_uint_t unichar_charlen(const char *str, mp_uint_t len) {
pythontech 0:5868e8752d44 114 #if MICROPY_PY_BUILTINS_STR_UNICODE
pythontech 0:5868e8752d44 115 mp_uint_t charlen = 0;
pythontech 0:5868e8752d44 116 for (const char *top = str + len; str < top; ++str) {
pythontech 0:5868e8752d44 117 if (!UTF8_IS_CONT(*str)) {
pythontech 0:5868e8752d44 118 ++charlen;
pythontech 0:5868e8752d44 119 }
pythontech 0:5868e8752d44 120 }
pythontech 0:5868e8752d44 121 return charlen;
pythontech 0:5868e8752d44 122 #else
pythontech 0:5868e8752d44 123 return len;
pythontech 0:5868e8752d44 124 #endif
pythontech 0:5868e8752d44 125 }
pythontech 0:5868e8752d44 126
pythontech 0:5868e8752d44 127 // Be aware: These unichar_is* functions are actually ASCII-only!
pythontech 0:5868e8752d44 128 bool unichar_isspace(unichar c) {
pythontech 0:5868e8752d44 129 return c < 128 && (attr[c] & FL_SPACE) != 0;
pythontech 0:5868e8752d44 130 }
pythontech 0:5868e8752d44 131
pythontech 0:5868e8752d44 132 bool unichar_isalpha(unichar c) {
pythontech 0:5868e8752d44 133 return c < 128 && (attr[c] & FL_ALPHA) != 0;
pythontech 0:5868e8752d44 134 }
pythontech 0:5868e8752d44 135
pythontech 0:5868e8752d44 136 bool unichar_isprint(unichar c) {
pythontech 0:5868e8752d44 137 return c < 128 && (attr[c] & FL_PRINT) != 0;
pythontech 0:5868e8752d44 138 }
pythontech 0:5868e8752d44 139
pythontech 0:5868e8752d44 140 bool unichar_isdigit(unichar c) {
pythontech 0:5868e8752d44 141 return c < 128 && (attr[c] & FL_DIGIT) != 0;
pythontech 0:5868e8752d44 142 }
pythontech 0:5868e8752d44 143
pythontech 0:5868e8752d44 144 bool unichar_isxdigit(unichar c) {
pythontech 0:5868e8752d44 145 return c < 128 && (attr[c] & FL_XDIGIT) != 0;
pythontech 0:5868e8752d44 146 }
pythontech 0:5868e8752d44 147
pythontech 0:5868e8752d44 148 bool unichar_isident(unichar c) {
pythontech 0:5868e8752d44 149 return c < 128 && ((attr[c] & (FL_ALPHA | FL_DIGIT)) != 0 || c == '_');
pythontech 0:5868e8752d44 150 }
pythontech 0:5868e8752d44 151
pythontech 0:5868e8752d44 152 bool unichar_isupper(unichar c) {
pythontech 0:5868e8752d44 153 return c < 128 && (attr[c] & FL_UPPER) != 0;
pythontech 0:5868e8752d44 154 }
pythontech 0:5868e8752d44 155
pythontech 0:5868e8752d44 156 bool unichar_islower(unichar c) {
pythontech 0:5868e8752d44 157 return c < 128 && (attr[c] & FL_LOWER) != 0;
pythontech 0:5868e8752d44 158 }
pythontech 0:5868e8752d44 159
pythontech 0:5868e8752d44 160 unichar unichar_tolower(unichar c) {
pythontech 0:5868e8752d44 161 if (unichar_isupper(c)) {
pythontech 0:5868e8752d44 162 return c + 0x20;
pythontech 0:5868e8752d44 163 }
pythontech 0:5868e8752d44 164 return c;
pythontech 0:5868e8752d44 165 }
pythontech 0:5868e8752d44 166
pythontech 0:5868e8752d44 167 unichar unichar_toupper(unichar c) {
pythontech 0:5868e8752d44 168 if (unichar_islower(c)) {
pythontech 0:5868e8752d44 169 return c - 0x20;
pythontech 0:5868e8752d44 170 }
pythontech 0:5868e8752d44 171 return c;
pythontech 0:5868e8752d44 172 }
pythontech 0:5868e8752d44 173
pythontech 0:5868e8752d44 174 mp_uint_t unichar_xdigit_value(unichar c) {
pythontech 0:5868e8752d44 175 // c is assumed to be hex digit
pythontech 0:5868e8752d44 176 mp_uint_t n = c - '0';
pythontech 0:5868e8752d44 177 if (n > 9) {
pythontech 0:5868e8752d44 178 n &= ~('a' - 'A');
pythontech 0:5868e8752d44 179 n -= ('A' - ('9' + 1));
pythontech 0:5868e8752d44 180 }
pythontech 0:5868e8752d44 181 return n;
pythontech 0:5868e8752d44 182 }