diff --git a/CVE-2022-29486.patch b/CVE-2022-29486.patch deleted file mode 100644 index 8cfe0a547c1bd573f68f686075b8077fb110835d..0000000000000000000000000000000000000000 --- a/CVE-2022-29486.patch +++ /dev/null @@ -1,23 +0,0 @@ -From 3070f11991cc2014685a28c0eaa1e033ffa8fe30 Mon Sep 17 00:00:00 2001 -From: "Hong, Yang A" -Date: Thu, 28 Apr 2022 10:11:32 +0000 -Subject: [PATCH] bugfix: fix overflow risk of strlen function - ---- - src/compiler/compiler.cpp | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp -index 6f993ffe..35f46b3f 100644 ---- a/src/compiler/compiler.cpp -+++ b/src/compiler/compiler.cpp -@@ -323,7 +323,8 @@ void addExpression(NG &ng, unsigned index, const char *expression, - } - - // Ensure that our pattern isn't too long (in characters). -- if (strlen(expression) > cc.grey.limitPatternLength) { -+ size_t maxlen = cc.grey.limitPatternLength + 1; -+ if (strnlen(expression, maxlen) >= maxlen) { - throw CompileError("Pattern length exceeds limit."); - } - diff --git a/Fix-build-error-on-x86_64.patch b/Fix-build-error-on-x86_64.patch deleted file mode 100644 index 58ec1fa8ab757573e9448eabc69ab5779209c7c3..0000000000000000000000000000000000000000 --- a/Fix-build-error-on-x86_64.patch +++ /dev/null @@ -1,25 +0,0 @@ -From f92e690190b51eb6bada384174887501f4c3f43f Mon Sep 17 00:00:00 2001 -From: wang_yue111 <648774160@qq.com> -Date: Sat, 24 Jul 2021 10:21:03 +0800 -Subject: [PATCH] fix build error on X86 - ---- - cmake/build_wrapper.sh | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/cmake/build_wrapper.sh b/cmake/build_wrapper.sh -index 1962813..895610c 100755 ---- a/cmake/build_wrapper.sh -+++ b/cmake/build_wrapper.sh -@@ -17,7 +17,7 @@ KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX) - LIBC_SO=$("$@" --print-file-name=libc.so.6) - cp ${KEEPSYMS_IN} ${KEEPSYMS} - # get all symbols from libc and turn them into patterns --nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ ]*\).*/^\1$/' >> ${KEEPSYMS} -+nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS} - # build the object - "$@" - # rename the symbols in the object --- -2.23.0 - diff --git a/backport-Fix-segfaults-on-allocation-failure.patch b/backport-Fix-segfaults-on-allocation-failure.patch deleted file mode 100644 index 041081c61150f0d51fc6c8825b5a04e007659ec2..0000000000000000000000000000000000000000 --- a/backport-Fix-segfaults-on-allocation-failure.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 7d644e7ba27eaadda753febf0b142faa9affbbca Mon Sep 17 00:00:00 2001 -From: hongyang7 -Date: Thu, 16 Dec 2021 19:02:17 +0800 -Subject: [PATCH] Fix segfaults on allocation failure (#4) - -Throw std::bad_alloc instead of returning nullptr from -ue2::AlignedAllocator. Allocators for STL containers are expected never -to return with an invalid pointer, and instead must throw on failure. -Violating this expectation can lead to invalid pointer dereferences. - -Co-authored-by: johanngan - -fixes github issue #317 (PR #320) ---- - src/util/alloc.h | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/src/util/alloc.h b/src/util/alloc.h -index de20c8d02..49b4a824d 100644 ---- a/src/util/alloc.h -+++ b/src/util/alloc.h -@@ -76,7 +76,11 @@ class AlignedAllocator { - - T *allocate(std::size_t size) const { - size_t alloc_size = size * sizeof(T); -- return static_cast(aligned_malloc_internal(alloc_size, N)); -+ T *ptr = static_cast(aligned_malloc_internal(alloc_size, N)); -+ if (!ptr) { -+ throw std::bad_alloc(); -+ } -+ return ptr; - } - - void deallocate(T *x, std::size_t) const noexcept { diff --git a/hyperscan-5.4.0.tar.gz b/hyperscan-5.4.0.tar.gz deleted file mode 100644 index 94a94cfe6bbdc02bd6eb4def8fc913635d594c7c..0000000000000000000000000000000000000000 Binary files a/hyperscan-5.4.0.tar.gz and /dev/null differ diff --git a/hyperscan-5.4.2.tar.gz b/hyperscan-5.4.2.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4c854e0c52cc12dc0f9bb85ec5a1660722817e90 Binary files /dev/null and b/hyperscan-5.4.2.tar.gz differ diff --git a/hyperscan-aarch64-support.patch b/hyperscan-aarch64-support.patch index 4ec64bfafca54765b0cc86185427a2ca57b1b6c6..a35c8f162854892a5bbabfaa106124abf19b1326 100644 --- a/hyperscan-aarch64-support.patch +++ b/hyperscan-aarch64-support.patch @@ -1,107 +1,48 @@ -From ddbbd2d1c012b53a1370927c065647e799368d0f Mon Sep 17 00:00:00 2001 +From e95491b3a2261aecdc5576a7e507b4f4ace88cbc Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Mon, 20 Jul 2020 17:20:15 +0800 Subject: [PATCH] Add aarch64 support +Signed-off-by: Liu Zixian --- - .travis.yml | 45 + - CMakeLists.txt | 117 +- - README.md | 39 +- - ThirdPartyNotice.md | 52 + + CMakeLists.txt | 108 +- cmake/config.h.in | 9 + cmake/platform.cmake | 13 +- cmake/ragel.cmake | 20 + src/crc32.c | 43 + - src/fdr/fdr.c | 136 +- + src/fdr/fdr.c | 136 ++- src/hs_valid_platform.c | 9 +- src/nfa/limex_exceptional.h | 22 +- src/nfa/limex_internal.h | 2 +- src/nfa/limex_native.c | 10 +- - src/nfa/shufti.c | 580 +- + src/nfa/shufti.c | 18 +- src/nfa/truffle.c | 10 +- - src/parser/Parser.cpp | 6397 ++++++++++++++++++++ - src/parser/control_verbs.cpp | 340 ++ + src/parser/control_verbs.cpp | 340 +++++++ src/rose/counting_miracle.h | 2 +- src/util/arch.h | 11 + - src/util/cpuid_flags.c | 9 +- + src/util/cpuid_flags.c | 6 + src/util/cpuid_flags.h | 2 + src/util/cpuid_inline.h | 17 +- src/util/intrinsics.h | 12 + src/util/popcount.h | 6 +- - src/util/simd_arm.h | 1069 ++++ - src/util/simd_types.h | 42 +- - src/util/simd_utils.h | 1389 +---- - src/util/simd_x86.h | 1334 ++++ - src/util/state_compress.c | 42 +- + src/util/simd_arm.h | 1069 ++++++++++++++++++++ + src/util/simd_types.h | 17 + + src/util/simd_utils.h | 13 + + src/util/simd_x86.h | 10 + tools/hscollider/CMakeLists.txt | 9 +- - tools/hscollider/ColliderCorporaParser.cpp | 474 ++ - unit/internal/simd_utils.cpp | 128 +- + tools/hscollider/ColliderCorporaParser.cpp | 474 +++++++++ + unit/internal/simd_utils.cpp | 2 +- util/CMakeLists.txt | 8 +- - util/ExpressionParser.cpp | 397 ++ - 34 files changed, 10994 insertions(+), 1801 deletions(-) - create mode 100644 .travis.yml - create mode 100644 ThirdPartyNotice.md - create mode 100644 src/parser/Parser.cpp + util/ExpressionParser.cpp | 397 ++++++++ + 29 files changed, 2717 insertions(+), 78 deletions(-) create mode 100644 src/parser/control_verbs.cpp create mode 100644 src/util/simd_arm.h - create mode 100644 src/util/simd_x86.h + create mode 100644 src/util/simd_utils.h create mode 100644 tools/hscollider/ColliderCorporaParser.cpp create mode 100644 util/ExpressionParser.cpp -diff --git a/.travis.yml b/.travis.yml -new file mode 100644 -index 0000000..7bfc3f1 ---- /dev/null -+++ b/.travis.yml -@@ -0,0 +1,45 @@ -+dist: bionic -+ -+os: -+ - linux -+ -+arch: -+ - arm64 -+ - x86 -+ -+addons: -+ apt: -+ update: true -+ hosts: -+ - node2 -+ -+language: -+ - c -+ -+script: -+ - cd .. -+ - sudo apt-get install libboost-dev -+ -+ - wget http://www.colm.net/files/ragel/ragel-6.10.tar.gz -+ - tar -zxf ragel-6.10.tar.gz -+ - cd ./ragel-6.10 -+ - sudo ./configure -+ - sudo make -j16 -+ - sudo make install -+ -+ - cd ../ -+ - wget https://ftp.pcre.org/pub/pcre/pcre-8.41.tar.gz -+ - tar -zxvf pcre-8.41.tar.gz -+ - sudo cp -rf ./pcre-8.41 ./hyperscan/pcre -+ -+ - cd ./hyperscan -+ - rm -rf build -+ - mkdir build -+ - cd build -+ - cmake .. -+ - make -j8 -+ - sudo ./bin/unit-hyperscan -+ - sudo ./bin/unit-chimera -+ - sudo ./bin/hscheck -s ./../tools/hscollider/test_cases/signatures/collider_tests.txt -+ - sudo make collide_quick_test -+ - sudo make collide_quick_test_block -\ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt -index 8bc6077..b5f8fb4 100644 +index bd6d2de..8dbcb72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ include (${CMAKE_MODULE_PATH}/boost.cmake) @@ -197,15 +138,7 @@ index 8bc6077..b5f8fb4 100644 endif() if(CMAKE_COMPILER_IS_GNUCC) -@@ -279,7 +301,6 @@ else() - set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -fno-omit-frame-pointer") - endif() - -- - if (CMAKE_C_COMPILER_ID MATCHES "Intel") - set(SKYLAKE_FLAG "-xCORE-AVX512") - else () -@@ -289,10 +310,18 @@ else() +@@ -289,10 +311,18 @@ else() endif() CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H) @@ -228,7 +161,7 @@ index 8bc6077..b5f8fb4 100644 CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN) CHECK_FUNCTION_EXISTS(_aligned_malloc HAVE__ALIGNED_MALLOC) -@@ -325,6 +354,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") +@@ -325,6 +355,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") (CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja"))) message (STATUS "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher") set (FAT_RUNTIME_REQUISITES FALSE) @@ -238,7 +171,7 @@ index 8bc6077..b5f8fb4 100644 else() include (${CMAKE_MODULE_PATH}/attrib.cmake) if (NOT HAS_C_ATTR_IFUNC) -@@ -337,7 +369,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") +@@ -337,7 +370,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") CMAKE_DEPENDENT_OPTION(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ${RELEASE_BUILD} "FAT_RUNTIME_REQUISITES" OFF) endif () @@ -249,7 +182,7 @@ index 8bc6077..b5f8fb4 100644 # testing a builtin takes a little more work CHECK_C_SOURCE_COMPILES("void *aa_test(void *x) { return __builtin_assume_aligned(x, 16);}\nint main(void) { return 0; }" HAVE_CC_BUILTIN_ASSUME_ALIGNED) -@@ -403,12 +437,6 @@ if (CXX_IGNORED_ATTR) +@@ -415,12 +450,6 @@ if (CXX_IGNORED_ATTR) set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-ignored-attributes") endif() @@ -262,7 +195,7 @@ index 8bc6077..b5f8fb4 100644 # note this for later # g++ doesn't have this flag but clang does CHECK_CXX_COMPILER_FLAG("-Wweak-vtables" CXX_WEAK_VTABLES) -@@ -463,6 +491,14 @@ else() +@@ -477,6 +506,14 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") endif() @@ -277,21 +210,7 @@ index 8bc6077..b5f8fb4 100644 add_subdirectory(util) add_subdirectory(doc/dev-reference) -@@ -552,14 +588,27 @@ set_source_files_properties( - PROPERTIES - COMPILE_FLAGS "${RAGEL_C_FLAGS}") - --ragelmaker(src/parser/Parser.rl) -+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386") -+ ragelmaker(src/parser/Parser.rl) -+endif() -+ -+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") -+ ragelcopyer(src/parser/Parser.rl) -+endif() - - set_source_files_properties( - ${CMAKE_BINARY_DIR}/src/parser/control_verbs.cpp +@@ -573,7 +610,14 @@ set_source_files_properties( PROPERTIES COMPILE_FLAGS "${RAGEL_C_FLAGS}") @@ -307,120 +226,6 @@ index 8bc6077..b5f8fb4 100644 SET(hs_HEADERS src/hs.h -diff --git a/README.md b/README.md -index 9f4c037..15c1ef6 100644 ---- a/README.md -+++ b/README.md -@@ -22,22 +22,41 @@ project repository. - - # Versioning - --The `master` branch on Github will always contain the most recent release of --Hyperscan. Each version released to `master` goes through QA and testing before --it is released; if you're a user, rather than a developer, this is the version --you should be using. -+The `master` branch on Github/kunpengcompute will always contain the most recent -+release of Intel Hyperscan. - --Further development towards the next release takes place on the `develop` --branch. -+The `aarch64` branch on Github/kunpengcompute will always contain the most recent -+release that supports the aarch64 architecture. The aarch64 branch was developed -+based on Intel hyperscan 5.2.1. Each version released to `aarch64` branch goes through -+QA and testing before it is released; if you're a user of aarch64, rather than a developer, -+this is the version you should be using. -+ -+# Porting -+Perform platform-specific different operations, including compilation options, -+detection specific header files, SIMD instruction judgment, and so on. -+ -+# Optimization -+Improve the Kunpeng platform by using the NEON instructions, inline assembly, -+data alignment, instruction alignment, memory data prefetching, static branch -+prediction, code structure optimization, etc. - - # Get Involved - - The official homepage for Hyperscan is at [www.hyperscan.io](https://www.hyperscan.io). - --If you have questions or comments, we encourage you to [join the mailing --list](https://lists.01.org/mailman/listinfo/hyperscan). Bugs can be filed by --sending email to the list, or by creating an issue on Github. -+`master` branch -+ -+If you have questions or comments, we encourage you to [join the mailing list] -+(https://lists.01.org/mailman/listinfo/hyperscan). To file a bug, you can send an email -+to the list, or create an issue on Github. - - If you wish to contact the Hyperscan team at Intel directly, without posting --publicly to the mailing list, send email to -+publicly to the mailing list, send an email to - [hyperscan@intel.com](mailto:hyperscan@intel.com). -+ -+`aarch64` branch -+ -+If you have questions or comments, we encourage you to create an issue on Github. -+ -+If you wish to contact the Huawei team directly, you can send an email to -+kunpengcompute@huawei.com. -diff --git a/ThirdPartyNotice.md b/ThirdPartyNotice.md -new file mode 100644 -index 0000000..f95cdac ---- /dev/null -+++ b/ThirdPartyNotice.md -@@ -0,0 +1,52 @@ -+# THIRD PARTY OPEN SOURCE SOFTWARE NOTICE -+Please note we provide an open source software notice for the third party open source software along with this software and/or this software component contributed by Huawei (in the following just “this SOFTWARE”). The open source software licenses are granted by the respective right holders. -+ -+# Warranty Disclaimer -+THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. -+ -+# Copyright Notice and License Texts -+Software: hyperscan 5.2.1 -+Copyright notice: -+ -+Copyright (c) 2015-2019, Intel Corporation -+Copyright (c) 2015-2017, Intel Corporation -+Copyright (c) 2015-2016, Intel Corporation -+Copyright (c) 2015, Intel Corporation -+Copyright (c) 2018, Intel Corporation -+Copyright (C) 2015 Intel Corporation. All rights reserved. -+Copyright -+Copyright |copy| 2015-2018, Intel Corporation. All rights reserved. -+Copyright (c) 2016, Intel Corporation -+Copyright (c) 2015-2018, Intel Corporation -+Copyright (c) 2016-2017, Intel Corporation -+Copyright (c) 2017-2018, Intel Corporation -+Copyright (c) 2016-2018, Intel Corporation -+Copyright (c) 2017, Intel Corporation -+Copyright 2008, Google Inc. -+Copyright 2005, Google Inc. -+Copyright 2007, Google Inc. -+Copyright (c) 2018-2019, Intel Corporation -+Copyright (c) 2016-2019, Intel Corporation -+Copyright (C) 2005-2009 Jongsoo Park -+Copyright (c) 2004-2006, Intel Corporation -+(C) As we report matches from (B) we interleave matches from the MPV if it -+Copyright (c) 2017-2019, Intel Corporation -+Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved -+Copyright (c) 2019, Intel Corporation -+ -+ -+License: BSD 3-Clause License -+ -+Copyright (c) , -+All rights reserved. -+ -+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -+ -+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -+ -+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -+ -+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -+ -+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ diff --git a/cmake/config.h.in b/cmake/config.h.in index 5454643..336cf19 100644 --- a/cmake/config.h.in @@ -942,157 +747,17 @@ index f6f5809..8998830 100644 } } diff --git a/src/nfa/shufti.c b/src/nfa/shufti.c -index 09ffc0c..6231e61 100644 +index 09ffc0c..2cb74f0 100644 --- a/src/nfa/shufti.c +++ b/src/nfa/shufti.c -@@ -42,39 +42,36 @@ - #ifdef DEBUG - #include - --#define DUMP_MSK(_t) \ --static UNUSED \ --void dumpMsk##_t(m##_t msk) { \ -- u8 * mskAsU8 = (u8 *)&msk; \ -- for (unsigned i = 0; i < sizeof(msk); i++) { \ -- u8 c = mskAsU8[i]; \ -- for (int j = 0; j < 8; j++) { \ -- if ((c >> (7-j)) & 0x1) \ -- printf("1"); \ -- else \ -- printf("0"); \ -- } \ -- printf(" "); \ -- } \ --} \ --static UNUSED \ --void dumpMsk##_t##AsChars(m##_t msk) { \ -- u8 * mskAsU8 = (u8 *)&msk; \ -- for (unsigned i = 0; i < sizeof(msk); i++) { \ -- u8 c = mskAsU8[i]; \ -- if (isprint(c)) \ -- printf("%c",c); \ -- else \ -- printf("."); \ -- } \ --} -+#define DUMP_MSK(_t) \ -+ static UNUSED void dumpMsk##_t(m##_t msk) { \ -+ u8 *mskAsU8 = (u8 *)&msk; \ -+ for (unsigned i = 0; i < sizeof(msk); i++) { \ -+ u8 c = mskAsU8[i]; \ -+ for (int j = 0; j < 8; j++) { \ -+ if ((c >> (7 - j)) & 0x1) \ -+ printf("1"); \ -+ else \ -+ printf("0"); \ -+ } \ -+ printf(" "); \ -+ } \ -+ } \ -+ static UNUSED void dumpMsk##_t##AsChars(m##_t msk) { \ -+ u8 *mskAsU8 = (u8 *)&msk; \ -+ for (unsigned i = 0; i < sizeof(msk); i++) { \ -+ u8 c = mskAsU8[i]; \ -+ if (isprint(c)) \ -+ printf("%c", c); \ -+ else \ -+ printf("."); \ -+ } \ -+ } - - #endif - - /** \brief Naive byte-by-byte implementation. */ --static really_inline --const u8 *shuftiFwdSlow(const u8 *lo, const u8 *hi, const u8 *buf, -- const u8 *buf_end) { -+static really_inline const u8 *shuftiFwdSlow(const u8 *lo, const u8 *hi, -+ const u8 *buf, const u8 *buf_end) { - assert(buf < buf_end); - - for (; buf < buf_end; ++buf) { -@@ -87,9 +84,8 @@ const u8 *shuftiFwdSlow(const u8 *lo, const u8 *hi, const u8 *buf, - } - - /** \brief Naive byte-by-byte implementation. */ --static really_inline --const u8 *shuftiRevSlow(const u8 *lo, const u8 *hi, const u8 *buf, -- const u8 *buf_end) { -+static really_inline const u8 *shuftiRevSlow(const u8 *lo, const u8 *hi, -+ const u8 *buf, const u8 *buf_end) { - assert(buf < buf_end); - - for (buf_end--; buf_end >= buf; buf_end--) { -@@ -111,25 +107,33 @@ DUMP_MSK(128) - #define GET_LO_4(chars) and128(chars, low4bits) - #define GET_HI_4(chars) rshift64_m128(andnot128(low4bits, chars), 4) - --static really_inline --u32 block(m128 mask_lo, m128 mask_hi, m128 chars, const m128 low4bits, -- const m128 compare) { -- m128 c_lo = pshufb_m128(mask_lo, GET_LO_4(chars)); -- m128 c_hi = pshufb_m128(mask_hi, GET_HI_4(chars)); -- m128 t = and128(c_lo, c_hi); -+static really_inline u32 block(m128 mask_lo, m128 mask_hi, m128 chars, -+ const m128 low4bits, const m128 compare) { -+ m128 c_lo = pshufb_m128(mask_lo, GET_LO_4(chars)); -+ m128 c_hi = pshufb_m128(mask_hi, GET_HI_4(chars)); -+ m128 t = and128(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk128AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk128(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk128(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk128(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk128(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk128AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk128(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk128(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk128(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk128(t); -+ printf("\n"); - #endif - return movemask128(eq128(t, compare)); - } - --static really_inline --const u8 *firstMatch(const u8 *buf, u32 z) { -+static really_inline const u8 *firstMatch(const u8 *buf, u32 z) { - if (unlikely(z != 0xffff)) { - u32 pos = ctz32(~z & 0xffff); - assert(pos < 16); -@@ -139,9 +143,9 @@ const u8 *firstMatch(const u8 *buf, u32 z) { - } - } - --static really_inline --const u8 *fwdBlock(m128 mask_lo, m128 mask_hi, m128 chars, const u8 *buf, -- const m128 low4bits, const m128 zeroes) { -+static really_inline const u8 *fwdBlock(m128 mask_lo, m128 mask_hi, m128 chars, -+ const u8 *buf, const m128 low4bits, -+ const m128 zeroes) { - u32 z = block(mask_lo, mask_hi, chars, low4bits, zeroes); - - return firstMatch(buf, z); -@@ -153,13 +157,13 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, +@@ -153,13 +153,13 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, assert(buf < buf_end); // Slow path for small cases. - if (buf_end - buf < 16) { -- return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, -- buf, buf_end); + if (unlikely(buf_end - buf < 16)) { -+ return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, -+ buf_end); + return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, + buf, buf_end); } const m128 zeroes = zeroes128(); @@ -1101,7 +766,7 @@ index 09ffc0c..6231e61 100644 const u8 *rv; size_t min = (size_t)buf % 16; -@@ -179,6 +183,11 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, +@@ -179,6 +179,11 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *last_block = buf_end - 16; while (buf < last_block) { m128 lchars = load128(buf); @@ -1113,71 +778,7 @@ index 09ffc0c..6231e61 100644 rv = fwdBlock(mask_lo, mask_hi, lchars, buf, low4bits, zeroes); if (rv) { return rv; -@@ -198,10 +207,11 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf_end; - } - --static really_inline --const u8 *lastMatch(const u8 *buf, m128 t, m128 compare) { -+static really_inline const u8 *lastMatch(const u8 *buf, m128 t, m128 compare) { - #ifdef DEBUG -- DEBUG_PRINTF("confirming match in:"); dumpMsk128(t); printf("\n"); -+ DEBUG_PRINTF("confirming match in:"); -+ dumpMsk128(t); -+ printf("\n"); - #endif - - u32 z = movemask128(eq128(t, compare)); -@@ -215,20 +225,29 @@ const u8 *lastMatch(const u8 *buf, m128 t, m128 compare) { - } - } - -- --static really_inline --const u8 *revBlock(m128 mask_lo, m128 mask_hi, m128 chars, const u8 *buf, -- const m128 low4bits, const m128 zeroes) { -- m128 c_lo = pshufb_m128(mask_lo, GET_LO_4(chars)); -- m128 c_hi = pshufb_m128(mask_hi, GET_HI_4(chars)); -- m128 t = and128(c_lo, c_hi); -+static really_inline const u8 *revBlock(m128 mask_lo, m128 mask_hi, m128 chars, -+ const u8 *buf, const m128 low4bits, -+ const m128 zeroes) { -+ m128 c_lo = pshufb_m128(mask_lo, GET_LO_4(chars)); -+ m128 c_hi = pshufb_m128(mask_hi, GET_HI_4(chars)); -+ m128 t = and128(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk128AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk128(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk128(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk128(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk128(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk128AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk128(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk128(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk128(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk128(t); -+ printf("\n"); - #endif - - return lastMatch(buf, t, zeroes); -@@ -241,12 +260,12 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - - // Slow path for small cases. - if (buf_end - buf < 16) { -- return shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, -- buf, buf_end); -+ return shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, -+ buf_end); +@@ -246,7 +251,7 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, } const m128 zeroes = zeroes128(); @@ -1186,708 +787,27 @@ index 09ffc0c..6231e61 100644 const u8 *rv; assert(buf_end - buf >= 16); -@@ -283,32 +302,48 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf - 1; - } - --static really_inline --const u8 *fwdBlock2(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi, -- m128 chars, const u8 *buf, const m128 low4bits, -- const m128 ones) { -+static really_inline const u8 *fwdBlock2(m128 mask1_lo, m128 mask1_hi, -+ m128 mask2_lo, m128 mask2_hi, -+ m128 chars, const u8 *buf, -+ const m128 low4bits, const m128 ones) { - m128 chars_lo = GET_LO_4(chars); - m128 chars_hi = GET_HI_4(chars); -- m128 c_lo = pshufb_m128(mask1_lo, chars_lo); -- m128 c_hi = pshufb_m128(mask1_hi, chars_hi); -- m128 t = or128(c_lo, c_hi); -+ m128 c_lo = pshufb_m128(mask1_lo, chars_lo); -+ m128 c_hi = pshufb_m128(mask1_hi, chars_hi); -+ m128 t = or128(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk128AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk128(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk128(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk128(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk128(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk128AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk128(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk128(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk128(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk128(t); -+ printf("\n"); - #endif - -- m128 c2_lo = pshufb_m128(mask2_lo, chars_lo); -- m128 c2_hi = pshufb_m128(mask2_hi, chars_hi); -- m128 t2 = or128(t, rshiftbyte_m128(or128(c2_lo, c2_hi), 1)); -+ m128 c2_lo = pshufb_m128(mask2_lo, chars_lo); -+ m128 c2_hi = pshufb_m128(mask2_hi, chars_hi); -+ m128 t2 = or128(t, rshiftbyte_m128(or128(c2_lo, c2_hi), 1)); - - #ifdef DEBUG -- DEBUG_PRINTF(" c2_lo: "); dumpMsk128(c2_lo); printf("\n"); -- DEBUG_PRINTF(" c2_hi: "); dumpMsk128(c2_hi); printf("\n"); -- DEBUG_PRINTF(" t2: "); dumpMsk128(t2); printf("\n"); -+ DEBUG_PRINTF(" c2_lo: "); -+ dumpMsk128(c2_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c2_hi: "); -+ dumpMsk128(c2_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t2: "); -+ dumpMsk128(t2); -+ printf("\n"); - #endif - - u32 z = movemask128(eq128(t2, ones)); -@@ -316,19 +351,18 @@ const u8 *fwdBlock2(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi, - return firstMatch(buf, z); - } - --const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, -- m128 mask2_lo, m128 mask2_hi, -- const u8 *buf, const u8 *buf_end) { -+const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, -+ m128 mask2_hi, const u8 *buf, const u8 *buf_end) { +@@ -320,7 +325,7 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, + m128 mask2_lo, m128 mask2_hi, + const u8 *buf, const u8 *buf_end) { const m128 ones = ones128(); - const m128 low4bits = _mm_set1_epi8(0xf); + const m128 low4bits = set16x8(0xf); const u8 *rv; size_t min = (size_t)buf % 16; - - // Preconditioning: most of the time our buffer won't be aligned. - m128 chars = loadu128(buf); -- rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, -- chars, buf, low4bits, ones); -+ rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars, buf, low4bits, -+ ones); - if (rv) { - return rv; - } -@@ -340,8 +374,13 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, +@@ -340,6 +345,11 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, const u8 *last_block = buf_end - 16; while (buf < last_block) { m128 lchars = load128(buf); -- rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, -- lchars, buf, low4bits, ones); + +#if defined(HAVE_NEON) + __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(buf + 256))); +#endif + -+ rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, lchars, buf, -+ low4bits, ones); - if (rv) { - return rv; - } -@@ -351,8 +390,8 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, - // Use an unaligned load to mop up the last 16 bytes and get an accurate - // picture to buf_end. - chars = loadu128(buf_end - 16); -- rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, -- chars, buf_end - 16, low4bits, ones); -+ rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, chars, buf_end - 16, -+ low4bits, ones); - if (rv) { - return rv; - } -@@ -370,26 +409,34 @@ DUMP_MSK(256) - #define GET_LO_4(chars) and256(chars, low4bits) - #define GET_HI_4(chars) rshift64_m256(andnot256(low4bits, chars), 4) - --static really_inline --u32 block(m256 mask_lo, m256 mask_hi, m256 chars, const m256 low4bits, -- const m256 compare) { -- m256 c_lo = pshufb_m256(mask_lo, GET_LO_4(chars)); -- m256 c_hi = pshufb_m256(mask_hi, GET_HI_4(chars)); -+static really_inline u32 block(m256 mask_lo, m256 mask_hi, m256 chars, -+ const m256 low4bits, const m256 compare) { -+ m256 c_lo = pshufb_m256(mask_lo, GET_LO_4(chars)); -+ m256 c_hi = pshufb_m256(mask_hi, GET_HI_4(chars)); - m256 t = and256(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk256AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk256(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk256(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk256(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk256(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk256AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk256(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk256(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk256(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk256(t); -+ printf("\n"); - #endif - - return movemask256(eq256(t, compare)); - } - --static really_inline --const u8 *firstMatch(const u8 *buf, u32 z) { -+static really_inline const u8 *firstMatch(const u8 *buf, u32 z) { - DEBUG_PRINTF("z 0x%08x\n", z); - if (unlikely(z != 0xffffffff)) { - u32 pos = ctz32(~z); -@@ -401,9 +448,8 @@ const u8 *firstMatch(const u8 *buf, u32 z) { - } - } - --static really_inline --const u8 *fwdBlockShort(m256 mask, m128 chars, const u8 *buf, -- const m256 low4bits) { -+static really_inline const u8 * -+fwdBlockShort(m256 mask, m128 chars, const u8 *buf, const m256 low4bits) { - // do the hi and lo shuffles in the one avx register - m256 c = combine2x128(rshift64_m128(chars, 4), chars); - c = and256(c, low4bits); -@@ -415,9 +461,9 @@ const u8 *fwdBlockShort(m256 mask, m128 chars, const u8 *buf, - return firstMatch(buf, z); - } - --static really_inline --const u8 *shuftiFwdShort(m128 mask_lo, m128 mask_hi, const u8 *buf, -- const u8 *buf_end, const m256 low4bits) { -+static really_inline const u8 *shuftiFwdShort(m128 mask_lo, m128 mask_hi, -+ const u8 *buf, const u8 *buf_end, -+ const m256 low4bits) { - // run shufti over two overlapping 16-byte unaligned reads - const m256 mask = combine2x128(mask_hi, mask_lo); - m128 chars = loadu128(buf); -@@ -434,9 +480,9 @@ const u8 *shuftiFwdShort(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf_end; - } - --static really_inline --const u8 *fwdBlock(m256 mask_lo, m256 mask_hi, m256 chars, const u8 *buf, -- const m256 low4bits, const m256 zeroes) { -+static really_inline const u8 *fwdBlock(m256 mask_lo, m256 mask_hi, m256 chars, -+ const u8 *buf, const m256 low4bits, -+ const m256 zeroes) { - u32 z = block(mask_lo, mask_hi, chars, low4bits, zeroes); - - return firstMatch(buf, z); -@@ -451,8 +497,8 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - - // Slow path for small cases. - if (buf_end - buf < 16) { -- return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, -- buf, buf_end); -+ return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, -+ buf_end); - } - - const m256 low4bits = set32x8(0xf); -@@ -483,7 +529,8 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - const u8 *last_block = buf_end - 32; - while (buf < last_block) { - m256 lchars = load256(buf); -- rv = fwdBlock(wide_mask_lo, wide_mask_hi, lchars, buf, low4bits, zeroes); -+ rv = -+ fwdBlock(wide_mask_lo, wide_mask_hi, lchars, buf, low4bits, zeroes); - if (rv) { - return rv; - } -@@ -494,7 +541,8 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - // picture to buf_end. - assert(buf <= buf_end && buf >= buf_end - 32); - chars = loadu256(buf_end - 32); -- rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, buf_end - 32, low4bits, zeroes); -+ rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, buf_end - 32, low4bits, -+ zeroes); - if (rv) { - return rv; - } -@@ -502,8 +550,7 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf_end; - } - --static really_inline --const u8 *lastMatch(const u8 *buf, u32 z) { -+static really_inline const u8 *lastMatch(const u8 *buf, u32 z) { - if (unlikely(z != 0xffffffff)) { - u32 pos = clz32(~z); - DEBUG_PRINTF("buf=%p, pos=%u\n", buf, pos); -@@ -513,28 +560,37 @@ const u8 *lastMatch(const u8 *buf, u32 z) { - } - } - --static really_inline --const u8 *revBlock(m256 mask_lo, m256 mask_hi, m256 chars, const u8 *buf, -- const m256 low4bits, const m256 zeroes) { -- m256 c_lo = pshufb_m256(mask_lo, GET_LO_4(chars)); -- m256 c_hi = pshufb_m256(mask_hi, GET_HI_4(chars)); -- m256 t = and256(c_lo, c_hi); -+static really_inline const u8 *revBlock(m256 mask_lo, m256 mask_hi, m256 chars, -+ const u8 *buf, const m256 low4bits, -+ const m256 zeroes) { -+ m256 c_lo = pshufb_m256(mask_lo, GET_LO_4(chars)); -+ m256 c_hi = pshufb_m256(mask_hi, GET_HI_4(chars)); -+ m256 t = and256(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk256AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk256(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk256(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk256(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk256(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk256AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk256(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk256(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk256(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk256(t); -+ printf("\n"); - #endif - - u32 z = movemask256(eq256(t, zeroes)); - return lastMatch(buf, z); - } - --static really_inline --const u8 *revBlockShort(m256 mask, m128 chars, const u8 *buf, -- const m256 low4bits) { -+static really_inline const u8 * -+revBlockShort(m256 mask, m128 chars, const u8 *buf, const m256 low4bits) { - // do the hi and lo shuffles in the one avx register - m256 c = combine2x128(rshift64_m128(chars, 4), chars); - c = and256(c, low4bits); -@@ -546,9 +602,9 @@ const u8 *revBlockShort(m256 mask, m128 chars, const u8 *buf, - return lastMatch(buf, z); - } - --static really_inline --const u8 *shuftiRevShort(m128 mask_lo, m128 mask_hi, const u8 *buf, -- const u8 *buf_end, const m256 low4bits) { -+static really_inline const u8 *shuftiRevShort(m128 mask_lo, m128 mask_hi, -+ const u8 *buf, const u8 *buf_end, -+ const m256 low4bits) { - // run shufti over two overlapping 16-byte unaligned reads - const m256 mask = combine2x128(mask_hi, mask_lo); - -@@ -566,7 +622,6 @@ const u8 *shuftiRevShort(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf - 1; - } - -- - /* takes 128 bit masks, but operates on 256 bits of data */ - const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - const u8 *buf_end) { -@@ -575,8 +630,8 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - - // Slow path for small cases. - if (buf_end - buf < 16) { -- return shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, -- buf, buf_end); -+ return shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, -+ buf_end); - } - - const m256 low4bits = set32x8(0xf); -@@ -594,7 +649,8 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - - // Preconditioning: most of the time our buffer won't be aligned. - m256 chars = loadu256(buf_end - 32); -- rv = revBlock(wide_mask_lo, wide_mask_hi, chars, buf_end - 32, low4bits, zeroes); -+ rv = revBlock(wide_mask_lo, wide_mask_hi, chars, buf_end - 32, low4bits, -+ zeroes); - if (rv) { - return rv; - } -@@ -606,7 +662,8 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - while (buf_end > last_block) { - buf_end -= 32; - m256 lchars = load256(buf_end); -- rv = revBlock(wide_mask_lo, wide_mask_hi, lchars, buf_end, low4bits, zeroes); -+ rv = revBlock(wide_mask_lo, wide_mask_hi, lchars, buf_end, low4bits, -+ zeroes); - if (rv) { - return rv; - } -@@ -623,42 +680,58 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf, - return buf - 1; - } - --static really_inline --const u8 *fwdBlock2(m256 mask1_lo, m256 mask1_hi, m256 mask2_lo, m256 mask2_hi, -- m256 chars, const u8 *buf, const m256 low4bits, -- const m256 ones) { -+static really_inline const u8 *fwdBlock2(m256 mask1_lo, m256 mask1_hi, -+ m256 mask2_lo, m256 mask2_hi, -+ m256 chars, const u8 *buf, -+ const m256 low4bits, const m256 ones) { - DEBUG_PRINTF("buf %p\n", buf); - m256 chars_lo = GET_LO_4(chars); - m256 chars_hi = GET_HI_4(chars); -- m256 c_lo = pshufb_m256(mask1_lo, chars_lo); -- m256 c_hi = pshufb_m256(mask1_hi, chars_hi); -- m256 t = or256(c_lo, c_hi); -+ m256 c_lo = pshufb_m256(mask1_lo, chars_lo); -+ m256 c_hi = pshufb_m256(mask1_hi, chars_hi); -+ m256 t = or256(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk256AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk256(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk256(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk256(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk256(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk256AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk256(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk256(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk256(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk256(t); -+ printf("\n"); - #endif - -- m256 c2_lo = pshufb_m256(mask2_lo, chars_lo); -- m256 c2_hi = pshufb_m256(mask2_hi, chars_hi); -+ m256 c2_lo = pshufb_m256(mask2_lo, chars_lo); -+ m256 c2_hi = pshufb_m256(mask2_hi, chars_hi); - m256 t2 = or256(t, rshift128_m256(or256(c2_lo, c2_hi), 1)); - - #ifdef DEBUG -- DEBUG_PRINTF(" c2_lo: "); dumpMsk256(c2_lo); printf("\n"); -- DEBUG_PRINTF(" c2_hi: "); dumpMsk256(c2_hi); printf("\n"); -- DEBUG_PRINTF(" t2: "); dumpMsk256(t2); printf("\n"); -+ DEBUG_PRINTF(" c2_lo: "); -+ dumpMsk256(c2_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c2_hi: "); -+ dumpMsk256(c2_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t2: "); -+ dumpMsk256(t2); -+ printf("\n"); - #endif - u32 z = movemask256(eq256(t2, ones)); - - return firstMatch(buf, z); - } - --static really_inline --const u8 *fwdBlockShort2(m256 mask1, m256 mask2, m128 chars, const u8 *buf, -- const m256 low4bits) { -+static really_inline const u8 *fwdBlockShort2(m256 mask1, m256 mask2, -+ m128 chars, const u8 *buf, -+ const m256 low4bits) { - // do the hi and lo shuffles in the one avx register - m256 c = combine2x128(rshift64_m128(chars, 4), chars); - c = and256(c, low4bits); -@@ -672,9 +745,10 @@ const u8 *fwdBlockShort2(m256 mask1, m256 mask2, m128 chars, const u8 *buf, - return firstMatch(buf, z); - } - --static really_inline --const u8 *shuftiDoubleShort(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, -- m128 mask2_hi, const u8 *buf, const u8 *buf_end) { -+static really_inline const u8 *shuftiDoubleShort(m128 mask1_lo, m128 mask1_hi, -+ m128 mask2_lo, m128 mask2_hi, -+ const u8 *buf, -+ const u8 *buf_end) { - DEBUG_PRINTF("buf %p len %zu\n", buf, buf_end - buf); - const m256 low4bits = set32x8(0xf); - // run shufti over two overlapping 16-byte unaligned reads -@@ -695,9 +769,8 @@ const u8 *shuftiDoubleShort(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, - } - - /* takes 128 bit masks, but operates on 256 bits of data */ --const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, -- m128 mask2_lo, m128 mask2_hi, -- const u8 *buf, const u8 *buf_end) { -+const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, -+ m128 mask2_hi, const u8 *buf, const u8 *buf_end) { - /* we should always have at least 16 bytes */ - assert(buf_end - buf >= 16); - DEBUG_PRINTF("buf %p len %zu\n", buf, buf_end - buf); -@@ -731,8 +804,8 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, - const u8 *last_block = buf_end - 32; - while (buf < last_block) { - m256 lchars = load256(buf); -- rv = fwdBlock2(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, -- lchars, buf, low4bits, ones); -+ rv = fwdBlock2(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, -+ wide_mask2_hi, lchars, buf, low4bits, ones); + rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi, + lchars, buf, low4bits, ones); if (rv) { - return rv; - } -@@ -757,26 +830,34 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, - DUMP_MSK(512) - #endif - --static really_inline --u64a block(m512 mask_lo, m512 mask_hi, m512 chars, const m512 low4bits, -- const m512 compare) { -+static really_inline u64a block(m512 mask_lo, m512 mask_hi, m512 chars, -+ const m512 low4bits, const m512 compare) { - m512 c_lo = pshufb_m512(mask_lo, and512(chars, low4bits)); -- m512 c_hi = pshufb_m512(mask_hi, -- rshift64_m512(andnot512(low4bits, chars), 4)); -+ m512 c_hi = -+ pshufb_m512(mask_hi, rshift64_m512(andnot512(low4bits, chars), 4)); - m512 t = and512(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk512AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk512(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk512(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk512(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk512(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk512AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk512(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk512(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk512(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk512(t); -+ printf("\n"); - #endif - - return eq512mask(t, compare); - } --static really_inline --const u8 *firstMatch64(const u8 *buf, u64a z) { -+static really_inline const u8 *firstMatch64(const u8 *buf, u64a z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = ctz64(~z); -@@ -788,18 +869,19 @@ const u8 *firstMatch64(const u8 *buf, u64a z) { - } - } - --static really_inline --const u8 *fwdBlock512(m512 mask_lo, m512 mask_hi, m512 chars, const u8 *buf, -- const m512 low4bits, const m512 zeroes) { -+static really_inline const u8 *fwdBlock512(m512 mask_lo, m512 mask_hi, -+ m512 chars, const u8 *buf, -+ const m512 low4bits, -+ const m512 zeroes) { - u64a z = block(mask_lo, mask_hi, chars, low4bits, zeroes); - - return firstMatch64(buf, z); - } - --static really_inline --const u8 *shortShufti512(m512 mask_lo, m512 mask_hi, const u8 *buf, -- const u8 *buf_end, const m512 low4bits, -- const m512 zeroes) { -+static really_inline const u8 *shortShufti512(m512 mask_lo, m512 mask_hi, -+ const u8 *buf, const u8 *buf_end, -+ const m512 low4bits, -+ const m512 zeroes) { - DEBUG_PRINTF("short shufti %p len %zu\n", buf, buf_end - buf); - uintptr_t len = buf_end - buf; - assert(len <= 64); -@@ -877,8 +959,7 @@ done: - return buf_end; - } - --static really_inline --const u8 *lastMatch64(const u8 *buf, u64a z) { -+static really_inline const u8 *lastMatch64(const u8 *buf, u64a z) { - DEBUG_PRINTF("z 0x%016llx\n", z); - if (unlikely(z != ~0ULL)) { - u32 pos = clz64(~z); -@@ -889,10 +970,10 @@ const u8 *lastMatch64(const u8 *buf, u64a z) { - } - } - --static really_inline --const u8 *rshortShufti512(m512 mask_lo, m512 mask_hi, const u8 *buf, -- const u8 *buf_end, const m512 low4bits, -- const m512 zeroes) { -+static really_inline const u8 *rshortShufti512(m512 mask_lo, m512 mask_hi, -+ const u8 *buf, const u8 *buf_end, -+ const m512 low4bits, -+ const m512 zeroes) { - DEBUG_PRINTF("short %p len %zu\n", buf, buf_end - buf); - uintptr_t len = buf_end - buf; - assert(len <= 64); -@@ -909,20 +990,31 @@ const u8 *rshortShufti512(m512 mask_lo, m512 mask_hi, const u8 *buf, - return lastMatch64(buf, z | ~k); - } - --static really_inline --const u8 *revBlock512(m512 mask_lo, m512 mask_hi, m512 chars, const u8 *buf, -- const m512 low4bits, const m512 zeroes) { -- m512 c_lo = pshufb_m512(mask_lo, and512(chars, low4bits)); -- m512 c_hi = pshufb_m512(mask_hi, -- rshift64_m512(andnot512(low4bits, chars), 4)); -- m512 t = and512(c_lo, c_hi); -+static really_inline const u8 *revBlock512(m512 mask_lo, m512 mask_hi, -+ m512 chars, const u8 *buf, -+ const m512 low4bits, -+ const m512 zeroes) { -+ m512 c_lo = pshufb_m512(mask_lo, and512(chars, low4bits)); -+ m512 c_hi = -+ pshufb_m512(mask_hi, rshift64_m512(andnot512(low4bits, chars), 4)); -+ m512 t = and512(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk512AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk512(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk512(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk512(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk512(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk512AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk512(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk512(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk512(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk512(t); -+ printf("\n"); - #endif - - u64a z = eq512mask(t, zeroes); -@@ -985,43 +1077,60 @@ done: - return buf - 1; - } - --static really_inline --const u8 *fwdBlock2(m512 mask1_lo, m512 mask1_hi, m512 mask2_lo, m512 mask2_hi, -- m512 chars, const u8 *buf, const m512 low4bits, -- const m512 ones, __mmask64 k) { -+static really_inline const u8 *fwdBlock2(m512 mask1_lo, m512 mask1_hi, -+ m512 mask2_lo, m512 mask2_hi, -+ m512 chars, const u8 *buf, -+ const m512 low4bits, const m512 ones, -+ __mmask64 k) { - DEBUG_PRINTF("buf %p %.64s\n", buf, buf); - m512 chars_lo = and512(chars, low4bits); - m512 chars_hi = rshift64_m512(andnot512(low4bits, chars), 4); -- m512 c_lo = maskz_pshufb_m512(k, mask1_lo, chars_lo); -- m512 c_hi = maskz_pshufb_m512(k, mask1_hi, chars_hi); -- m512 t = or512(c_lo, c_hi); -+ m512 c_lo = maskz_pshufb_m512(k, mask1_lo, chars_lo); -+ m512 c_hi = maskz_pshufb_m512(k, mask1_hi, chars_hi); -+ m512 t = or512(c_lo, c_hi); - - #ifdef DEBUG -- DEBUG_PRINTF(" chars: "); dumpMsk512AsChars(chars); printf("\n"); -- DEBUG_PRINTF(" char: "); dumpMsk512(chars); printf("\n"); -- DEBUG_PRINTF(" c_lo: "); dumpMsk512(c_lo); printf("\n"); -- DEBUG_PRINTF(" c_hi: "); dumpMsk512(c_hi); printf("\n"); -- DEBUG_PRINTF(" t: "); dumpMsk512(t); printf("\n"); -+ DEBUG_PRINTF(" chars: "); -+ dumpMsk512AsChars(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" char: "); -+ dumpMsk512(chars); -+ printf("\n"); -+ DEBUG_PRINTF(" c_lo: "); -+ dumpMsk512(c_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c_hi: "); -+ dumpMsk512(c_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t: "); -+ dumpMsk512(t); -+ printf("\n"); - #endif - -- m512 c2_lo = maskz_pshufb_m512(k, mask2_lo, chars_lo); -- m512 c2_hi = maskz_pshufb_m512(k, mask2_hi, chars_hi); -+ m512 c2_lo = maskz_pshufb_m512(k, mask2_lo, chars_lo); -+ m512 c2_hi = maskz_pshufb_m512(k, mask2_hi, chars_hi); - m512 t2 = or512(t, rshift128_m512(or512(c2_lo, c2_hi), 1)); - - #ifdef DEBUG -- DEBUG_PRINTF(" c2_lo: "); dumpMsk512(c2_lo); printf("\n"); -- DEBUG_PRINTF(" c2_hi: "); dumpMsk512(c2_hi); printf("\n"); -- DEBUG_PRINTF(" t2: "); dumpMsk512(t2); printf("\n"); -+ DEBUG_PRINTF(" c2_lo: "); -+ dumpMsk512(c2_lo); -+ printf("\n"); -+ DEBUG_PRINTF(" c2_hi: "); -+ dumpMsk512(c2_hi); -+ printf("\n"); -+ DEBUG_PRINTF(" t2: "); -+ dumpMsk512(t2); -+ printf("\n"); - #endif - u64a z = eq512mask(t2, ones); - - return firstMatch64(buf, z | ~k); - } - --static really_inline --const u8 *shortDoubleShufti512(m512 mask1_lo, m512 mask1_hi, m512 mask2_lo, -- m512 mask2_hi, const u8 *buf, const u8 *buf_end, -- const m512 low4bits, const m512 ones) { -+static really_inline const u8 * -+shortDoubleShufti512(m512 mask1_lo, m512 mask1_hi, m512 mask2_lo, m512 mask2_hi, -+ const u8 *buf, const u8 *buf_end, const m512 low4bits, -+ const m512 ones) { - DEBUG_PRINTF("short %p len %zu\n", buf, buf_end - buf); - uintptr_t len = buf_end - buf; - assert(len <= 64); -@@ -1038,9 +1147,8 @@ const u8 *shortDoubleShufti512(m512 mask1_lo, m512 mask1_hi, m512 mask2_lo, - } - - /* takes 128 bit masks, but operates on 512 bits of data */ --const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, -- m128 mask2_lo, m128 mask2_hi, -- const u8 *buf, const u8 *buf_end) { -+const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, -+ m128 mask2_hi, const u8 *buf, const u8 *buf_end) { - /* we should always have at least 16 bytes */ - assert(buf_end - buf >= 16); - DEBUG_PRINTF("buf %p len %zu\n", buf, buf_end - buf); diff --git a/src/nfa/truffle.c b/src/nfa/truffle.c index be6b312..c05d778 100644 --- a/src/nfa/truffle.c @@ -1930,15 +850,15 @@ index be6b312..c05d778 100644 return truffleMini(shuf_mask_lo_highclear, shuf_mask_lo_highset, buf, buf_end); } -diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp +diff --git a/src/parser/control_verbs.cpp b/src/parser/control_verbs.cpp new file mode 100644 -index 0000000..36d0e05 +index 0000000..482004d --- /dev/null -+++ b/src/parser/Parser.cpp -@@ -0,0 +1,6397 @@ ++++ b/src/parser/control_verbs.cpp +@@ -0,0 +1,340 @@ + +/* -+ * Copyright (c) 2015-2017, Intel Corporation ++ * Copyright (c) 2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: @@ -1965,1543 +885,174 @@ index 0000000..36d0e05 + * POSSIBILITY OF SUCH DAMAGE. + */ + -+/** \file -+ * \brief Parser code (generated with Ragel from Parser.rl). ++/** ++ * \file ++ * \brief Parser for control verbs that can occur at the beginning of a pattern. + */ + -+#include "config.h" ++#include "parser/control_verbs.h" + -+/* Parser.cpp is a built source, may not be in same dir as parser files */ -+#include "parser/Component.h" -+#include "parser/ComponentAlternation.h" -+#include "parser/ComponentAssertion.h" -+#include "parser/ComponentAtomicGroup.h" -+#include "parser/ComponentBackReference.h" -+#include "parser/ComponentBoundary.h" -+#include "parser/ComponentByte.h" -+#include "parser/ComponentClass.h" -+#include "parser/ComponentCondReference.h" -+#include "parser/ComponentEUS.h" -+#include "parser/ComponentEmpty.h" -+#include "parser/ComponentRepeat.h" -+#include "parser/ComponentSequence.h" -+#include "parser/ComponentWordBoundary.h" +#include "parser/Parser.h" -+#include "parser/check_refs.h" -+#include "parser/control_verbs.h" +#include "parser/parse_error.h" -+#include "ue2common.h" -+#include "util/compare.h" -+#include "util/flat_containers.h" -+#include "util/make_unique.h" -+#include "util/unicode_def.h" -+#include "util/verify_types.h" + -+#include -+#include -+#include +#include -+#include +#include -+#include -+#include + +using namespace std; + +namespace ue2 { + -+#define PUSH_SEQUENCE \ -+ do { \ -+ sequences.push_back(ExprState(currentSeq, (size_t)(ts - ptr), mode)); \ -+ } while (0) -+#define POP_SEQUENCE \ -+ do { \ -+ currentSeq = sequences.back().seq; \ -+ mode = sequences.back().mode; \ -+ sequences.pop_back(); \ -+ } while (0) -+ -+namespace { -+ -+/** \brief Structure representing current state as we're parsing (current -+ * sequence, current options). Stored in the 'sequences' vector. */ -+struct ExprState { -+ ExprState(ComponentSequence *seq_in, size_t offset, -+ const ParseMode &mode_in) -+ : seq(seq_in), seqOffset(offset), mode(mode_in) {} -+ -+ ComponentSequence *seq; //!< current sequence -+ size_t seqOffset; //!< offset seq was entered, for error reporting -+ ParseMode mode; //!< current mode flags -+}; -+ -+} // namespace -+ -+static unsigned parseAsDecimal(unsigned oct) { -+ // The input was parsed as octal, but should have been parsed as decimal. -+ // Deconstruct the octal number and reconstruct into decimal -+ unsigned ret = 0; -+ unsigned multiplier = 1; -+ while (oct) { -+ ret += (oct & 0x7) * multiplier; -+ oct >>= 3; -+ multiplier *= 10; -+ } -+ return ret; -+} -+ -+/** \brief Maximum value for a positive integer. We use INT_MAX, as that's what -+ * PCRE uses. */ -+static constexpr u32 MAX_NUMBER = INT_MAX; -+ -+static void pushDec(u32 *acc, char raw_digit) { -+ assert(raw_digit >= '0' && raw_digit <= '9'); -+ u32 digit_val = raw_digit - '0'; -+ -+ // Ensure that we don't overflow. -+ u64a val = ((u64a)*acc * 10) + digit_val; -+ if (val > MAX_NUMBER) { -+ throw LocatedParseError("Number is too big"); -+ } -+ -+ *acc = verify_u32(val); -+} -+ -+static void pushOct(u32 *acc, char raw_digit) { -+ assert(raw_digit >= '0' && raw_digit <= '7'); -+ u32 digit_val = raw_digit - '0'; -+ -+ // Ensure that we don't overflow. -+ u64a val = ((u64a)*acc * 8) + digit_val; -+ if (val > MAX_NUMBER) { -+ throw LocatedParseError("Number is too big"); -+ } -+ -+ *acc = verify_u32(val); -+} -+ -+static void throwInvalidRepeat(void) { -+ throw LocatedParseError("Invalid repeat"); -+} -+ -+static void throwInvalidUtf8(void) { -+ throw ParseError("Expression is not valid UTF-8."); -+} -+ -+/** -+ * Adds the given child component to the parent sequence, returning a pointer -+ * to the new (child) "current sequence". -+ */ -+static ComponentSequence *enterSequence(ComponentSequence *parent, -+ unique_ptr child) { -+ assert(parent); -+ assert(child); -+ -+ ComponentSequence *seq = child.get(); -+ parent->addComponent(move(child)); -+ return seq; -+} -+ -+static void addLiteral(ComponentSequence *currentSeq, char c, -+ const ParseMode &mode) { -+ if (mode.utf8 && mode.caseless) { -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ assert(cc); -+ cc->add(c); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } else { -+ currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); -+ } -+} -+ -+static void addEscaped(ComponentSequence *currentSeq, unichar accum, -+ const ParseMode &mode, const char *err_msg) { -+ if (mode.utf8) { -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ assert(cc); -+ cc->add(accum); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } else { -+ if (accum > 255) { -+ throw LocatedParseError(err_msg); -+ } -+ addLiteral(currentSeq, (char)accum, mode); -+ } -+} -+ -+static void addEscapedOctal(ComponentSequence *currentSeq, unichar accum, -+ const ParseMode &mode) { -+ addEscaped(currentSeq, accum, mode, "Octal value is greater than \\377"); -+} -+ -+static void addEscapedHex(ComponentSequence *currentSeq, unichar accum, -+ const ParseMode &mode) { -+ addEscaped(currentSeq, accum, mode, -+ "Hexadecimal value is greater than \\xFF"); -+} -+ -+#define SLASH_C_ERROR "\\c must be followed by an ASCII character" -+ -+static u8 decodeCtrl(char raw) { -+ if (raw & 0x80) { -+ throw LocatedParseError(SLASH_C_ERROR); -+ } -+ return mytoupper(raw) ^ 0x40; -+} -+ -+static unichar readUtf8CodePoint2c(const char *s) { -+ auto *ts = (const u8 *)s; -+ assert(ts[0] >= 0xc0 && ts[0] < 0xe0); -+ assert(ts[1] >= 0x80 && ts[1] < 0xc0); -+ unichar val = ts[0] & 0x1f; -+ val <<= 6; -+ val |= ts[1] & 0x3f; -+ DEBUG_PRINTF("utf8 %02hhx %02hhx ->\\x{%x}\n", ts[0], ts[1], val); -+ return val; -+} -+ -+static unichar readUtf8CodePoint3c(const char *s) { -+ auto *ts = (const u8 *)s; -+ assert(ts[0] >= 0xe0 && ts[0] < 0xf0); -+ assert(ts[1] >= 0x80 && ts[1] < 0xc0); -+ assert(ts[2] >= 0x80 && ts[2] < 0xc0); -+ unichar val = ts[0] & 0x0f; -+ val <<= 6; -+ val |= ts[1] & 0x3f; -+ val <<= 6; -+ val |= ts[2] & 0x3f; -+ DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], ts[1], ts[2], -+ val); -+ return val; -+} -+ -+static unichar readUtf8CodePoint4c(const char *s) { -+ auto *ts = (const u8 *)s; -+ assert(ts[0] >= 0xf0 && ts[0] < 0xf8); -+ assert(ts[1] >= 0x80 && ts[1] < 0xc0); -+ assert(ts[2] >= 0x80 && ts[2] < 0xc0); -+ assert(ts[3] >= 0x80 && ts[3] < 0xc0); -+ unichar val = ts[0] & 0x07; -+ val <<= 6; -+ val |= ts[1] & 0x3f; -+ val <<= 6; -+ val |= ts[2] & 0x3f; -+ val <<= 6; -+ val |= ts[3] & 0x3f; -+ DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], ts[1], -+ ts[2], ts[3], val); -+ return val; -+} -+ -+static const short _regex_actions[] = { -+ 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 7, 1, 8, -+ 1, 9, 1, 10, 1, 11, 1, 12, 1, 13, 1, 15, 1, 16, 1, -+ 17, 1, 18, 1, 19, 1, 20, 1, 21, 1, 22, 1, 23, 1, 24, -+ 1, 25, 1, 26, 1, 27, 1, 28, 1, 29, 1, 30, 1, 31, 1, -+ 32, 1, 33, 1, 34, 1, 35, 1, 36, 1, 37, 1, 38, 1, 39, -+ 1, 40, 1, 41, 1, 42, 1, 43, 1, 44, 1, 45, 1, 46, 1, -+ 47, 1, 48, 1, 49, 1, 50, 1, 51, 1, 52, 1, 53, 1, 54, -+ 1, 55, 1, 56, 1, 57, 1, 58, 1, 59, 1, 60, 1, 61, 1, -+ 62, 1, 63, 1, 64, 1, 65, 1, 66, 1, 67, 1, 68, 1, 69, -+ 1, 70, 1, 71, 1, 72, 1, 73, 1, 74, 1, 75, 1, 76, 1, -+ 77, 1, 78, 1, 79, 1, 80, 1, 81, 1, 82, 1, 83, 1, 84, -+ 1, 85, 1, 86, 1, 87, 1, 88, 1, 89, 1, 90, 1, 91, 1, -+ 92, 1, 93, 1, 94, 1, 95, 1, 96, 1, 97, 1, 98, 1, 99, -+ 1, 100, 1, 101, 1, 102, 1, 103, 1, 104, 1, 105, 1, 106, 1, -+ 107, 1, 108, 1, 109, 1, 110, 1, 111, 1, 112, 1, 113, 1, 114, -+ 1, 115, 1, 116, 1, 117, 1, 118, 1, 119, 1, 120, 1, 121, 1, -+ 122, 1, 123, 1, 124, 1, 125, 1, 126, 1, 127, 1, 128, 1, 129, -+ 1, 130, 1, 131, 1, 132, 1, 133, 1, 134, 1, 135, 1, 136, 1, -+ 137, 1, 138, 1, 139, 1, 140, 1, 141, 1, 142, 1, 143, 1, 144, -+ 1, 145, 1, 146, 1, 147, 1, 148, 1, 149, 1, 150, 1, 151, 1, -+ 152, 1, 153, 1, 154, 1, 155, 1, 156, 1, 157, 1, 158, 1, 159, -+ 1, 160, 1, 161, 1, 162, 1, 163, 1, 164, 1, 165, 1, 166, 1, -+ 167, 1, 168, 1, 169, 1, 170, 1, 171, 1, 172, 1, 173, 1, 174, -+ 1, 175, 1, 176, 1, 177, 1, 178, 1, 179, 1, 180, 1, 181, 1, -+ 182, 1, 183, 1, 184, 1, 185, 1, 186, 1, 187, 1, 188, 1, 189, -+ 1, 190, 1, 191, 1, 192, 1, 193, 1, 194, 1, 195, 1, 196, 1, -+ 197, 1, 198, 1, 199, 1, 200, 1, 201, 1, 202, 1, 203, 1, 204, -+ 1, 205, 1, 206, 1, 207, 1, 208, 1, 209, 1, 210, 1, 211, 1, -+ 212, 1, 213, 1, 214, 1, 215, 1, 216, 1, 217, 1, 218, 1, 219, -+ 1, 220, 1, 221, 1, 222, 1, 223, 1, 224, 1, 225, 1, 226, 1, -+ 227, 1, 228, 1, 229, 1, 230, 1, 231, 1, 232, 1, 233, 1, 234, -+ 1, 235, 1, 236, 1, 237, 1, 240, 1, 242, 1, 243, 1, 244, 1, -+ 245, 1, 246, 1, 247, 1, 248, 1, 249, 1, 250, 1, 251, 1, 252, -+ 1, 253, 1, 254, 1, 255, 1, 256, 1, 257, 1, 258, 1, 259, 1, -+ 260, 1, 261, 1, 262, 1, 263, 1, 264, 1, 265, 1, 266, 1, 267, -+ 1, 268, 1, 269, 1, 270, 1, 271, 1, 272, 1, 273, 1, 274, 1, -+ 275, 1, 276, 1, 277, 1, 278, 1, 279, 1, 280, 1, 281, 1, 282, -+ 1, 283, 1, 284, 1, 285, 1, 286, 1, 287, 1, 288, 1, 289, 1, -+ 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, -+ 1, 298, 1, 299, 1, 300, 1, 301, 1, 302, 1, 303, 1, 307, 1, -+ 308, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, -+ 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, -+ 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, -+ 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, 336, 1, 337, 1, -+ 338, 1, 342, 1, 343, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, -+ 1, 349, 1, 350, 1, 352, 1, 353, 1, 354, 1, 355, 1, 356, 1, -+ 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, -+ 1, 365, 1, 366, 1, 367, 1, 368, 1, 369, 1, 370, 1, 371, 1, -+ 372, 1, 373, 1, 374, 1, 375, 1, 376, 1, 377, 1, 378, 1, 379, -+ 1, 380, 1, 381, 1, 382, 1, 383, 1, 384, 1, 385, 1, 386, 1, -+ 387, 1, 388, 1, 389, 1, 390, 1, 391, 1, 392, 1, 393, 1, 394, -+ 1, 395, 1, 396, 1, 397, 1, 398, 1, 399, 1, 400, 1, 401, 1, -+ 402, 1, 403, 1, 404, 1, 405, 1, 406, 1, 407, 1, 408, 1, 409, -+ 1, 410, 1, 411, 1, 412, 1, 413, 1, 414, 1, 415, 1, 416, 1, -+ 417, 1, 418, 1, 419, 1, 420, 1, 421, 1, 422, 1, 423, 1, 424, -+ 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, -+ 432, 1, 433, 1, 434, 1, 435, 1, 436, 2, 3, 0, 2, 4, 5, -+ 2, 5, 1, 2, 9, 10, 2, 9, 238, 2, 9, 239, 2, 9, 339, -+ 2, 10, 1, 2, 10, 340, 2, 10, 341, 2, 11, 241, 2, 11, 351, -+ 2, 12, 241, 2, 12, 351, 2, 13, 241, 2, 13, 351, 2, 14, 375, -+ 2, 14, 376, 2, 25, 0, 2, 25, 3, 2, 25, 6, 2, 25, 14, -+ 3, 25, 5, 306, 3, 25, 10, 305, 3, 25, 14, 15, 4, 25, 9, -+ 304, 10}; -+ -+static const char _regex_cond_offsets[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 5, 6, -+ 7, 8, 9, 10, 11, 12, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, -+ 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, -+ 18, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -+ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -+ 21, 21, 21, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 28, -+ 29, 31, 31, 36, 36, 37, 38, 39, 44, 44, 45, 46, 47, 47}; -+ -+static const char _regex_cond_lengths[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 0, 5, 0, 1, 1, 1, 5, 0, 1, 1, 1, 0, 0}; -+ -+static const short _regex_cond_keys[] = { -+ -128, -65, -128, -65, -128, -65, -128, -65, -128, -65, -128, -65, -128, -65, -+ -128, -65, -128, -65, -128, -65, -128, -65, -128, -65, -128, -65, -64, -33, -+ -32, -17, -16, -9, -8, -1, 35, 35, -128, -65, -128, -65, -128, -65, -+ -128, -65, -64, -33, -32, -17, -16, -9, -8, -1, -128, -65, -128, -65, -+ -128, -65, 93, 93, 94, 94, -128, -65, -64, -33, -32, -17, -16, -9, -+ -8, -1, -128, -65, -128, -65, -128, -65, -128, -65, -64, -33, -32, -17, -+ -16, -9, -8, -1, -128, -65, -128, -65, -128, -65, 0}; -+ -+static const char _regex_cond_spaces[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -+ -+static const short _regex_key_offsets[] = { -+ 0, 0, 1, 23, 31, 39, 46, 54, 55, 63, 71, 79, -+ 86, 94, 97, 99, 108, 115, 123, 131, 134, 140, 148, 151, -+ 158, 165, 173, 180, 184, 191, 194, 197, 199, 202, 205, 207, -+ 210, 213, 215, 216, 218, 219, 227, 229, 232, 235, 236, 244, -+ 252, 260, 268, 275, 283, 290, 298, 305, 313, 315, 318, 325, -+ 329, 332, 335, 337, 339, 341, 342, 344, 345, 347, 349, 350, -+ 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, -+ 364, 365, 366, 369, 370, 371, 372, 373, 374, 375, 376, 377, -+ 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, -+ 390, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, -+ 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 415, 416, -+ 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 429, -+ 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, -+ 442, 443, 444, 445, 446, 447, 448, 450, 451, 452, 453, 454, -+ 455, 456, 457, 458, 459, 461, 462, 463, 464, 465, 466, 467, -+ 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, -+ 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, -+ 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, -+ 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, -+ 516, 517, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, -+ 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, -+ 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, -+ 553, 554, 555, 556, 557, 558, 559, 561, 562, 563, 564, 565, -+ 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, -+ 578, 579, 580, 582, 583, 584, 585, 586, 587, 588, 589, 590, -+ 591, 592, 593, 594, 595, 596, 597, 601, 602, 603, 604, 605, -+ 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, -+ 618, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 631, -+ 632, 633, 634, 635, 636, 637, 640, 641, 642, 643, 644, 645, -+ 646, 647, 648, 650, 651, 652, 653, 654, 655, 656, 658, 659, -+ 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, -+ 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, -+ 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, -+ 696, 697, 698, 699, 700, 701, 702, 704, 705, 706, 707, 708, -+ 709, 710, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, -+ 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, -+ 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, -+ 748, 749, 750, 752, 753, 754, 755, 756, 757, 758, 759, 760, -+ 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 773, -+ 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, -+ 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, -+ 798, 799, 800, 801, 802, 803, 805, 806, 807, 808, 809, 810, -+ 811, 812, 813, 814, 815, 816, 817, 820, 822, 823, 824, 825, -+ 826, 827, 828, 829, 830, 833, 834, 835, 836, 837, 838, 839, -+ 840, 841, 842, 843, 844, 845, 846, 847, 849, 850, 851, 853, -+ 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, -+ 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, -+ 880, 883, 885, 900, 903, 906, 908, 922, 927, 932, 936, 940, -+ 943, 946, 950, 954, 957, 960, 964, 968, 972, 975, 978, 982, -+ 986, 990, 994, 997, 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, -+ 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, 1060, 1063, 1066, 1070, -+ 1074, 1078, 1082, 1085, 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, -+ 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, 1148, 1151, 1155, 1159, -+ 1163, 1167, 1170, 1173, 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, -+ 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, 1238, 1241, 1245, 1249, -+ 1252, 1255, 1259, 1263, 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, -+ 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, 1325, 1329, 1333, 1336, -+ 1339, 1343, 1347, 1351, 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, -+ 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, 1414, 1418, 1421, 1424, -+ 1428, 1432, 1436, 1440, 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, -+ 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, 1502, 1506, 1509, 1512, -+ 1515, 1518, 1520, 1522, 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, -+ 1546, 1548, 1550, 1584, 1586, 1593, 1600, 1614, 1616, 1622, 1625, 1634, -+ 1635, 1638, 1641, 1648, 1650, 1652, 1654, 1657, 1702, 1704, 1706, 1710, -+ 1714, 1716, 1717, 1717, 1723, 1725, 1727, 1729, 1731, 1734, 1735, 1736, -+ 1743, 1749, 1755, 1757, 1759, 1761, 1763, 1765, 1767, 1768, 1771, 1794, -+ 1797, 1802, 1811, 1813, 1814, 1816, 1821, 1824, 1826, 1828, 1829, 1831, -+ 1841, 1847, 1848, 1853, 1857, 1865, 1867, 1876, 1880, 1881, 1882, 1886, -+ 1887, 1890, 1890, 1897, 1913, 1916, 1955, 1957, 1959, 1961, 1963, 1964, -+ 1964, 1965, 1966, 1973, 1979, 1985, 1987, 1989, 1991, 2000, 2002, 2015, -+ 2016, 2018, 2020, 2022, 2035, 2036, 2038, 2040, 2042, 2043}; -+ -+static const short _regex_trans_keys[] = { -+ 41, 33, 35, 38, 39, 40, 41, 43, 45, 58, 60, 61, 62, 63, -+ 67, 80, 105, 109, 115, 120, 123, 48, 57, 41, 95, 48, 57, 65, -+ 90, 97, 122, 39, 95, 48, 57, 65, 90, 97, 122, 95, 48, 57, -+ 65, 90, 97, 122, 39, 95, 48, 57, 65, 90, 97, 122, 41, 41, -+ 95, 48, 57, 65, 90, 97, 122, 41, 95, 48, 57, 65, 90, 97, -+ 122, 41, 95, 48, 57, 65, 90, 97, 122, 95, 48, 57, 65, 90, -+ 97, 122, 62, 95, 48, 57, 65, 90, 97, 122, 33, 60, 61, 33, -+ 61, 38, 41, 95, 48, 57, 65, 90, 97, 122, 95, 48, 57, 65, -+ 90, 97, 122, 41, 95, 48, 57, 65, 90, 97, 122, 41, 95, 48, -+ 57, 65, 90, 97, 122, 41, 48, 57, 41, 58, 105, 109, 115, 120, -+ 62, 95, 48, 57, 65, 90, 97, 122, 41, 48, 57, 95, 48, 57, -+ 65, 90, 97, 122, 95, 48, 57, 65, 90, 97, 122, 41, 95, 48, -+ 57, 65, 90, 97, 122, 95, 48, 57, 65, 90, 97, 122, 105, 109, -+ 115, 120, 41, 45, 58, 105, 109, 115, 120, 46, 92, 93, 46, 92, -+ 93, 46, 92, 58, 92, 93, 58, 92, 93, 58, 92, 61, 92, 93, -+ 61, 92, 93, 61, 92, 39, 48, 57, 62, 45, 95, 48, 57, 65, -+ 90, 97, 122, 48, 57, 125, 48, 57, 125, 48, 57, 125, 95, 125, -+ 48, 57, 65, 90, 97, 122, 95, 125, 48, 57, 65, 90, 97, 122, -+ 95, 125, 48, 57, 65, 90, 97, 122, 95, 125, 48, 57, 65, 90, -+ 97, 122, 95, 48, 57, 65, 90, 97, 122, 39, 95, 48, 57, 65, -+ 90, 97, 122, 95, 48, 57, 65, 90, 97, 122, 62, 95, 48, 57, -+ 65, 90, 97, 122, 95, 48, 57, 65, 90, 97, 122, 95, 125, 48, -+ 57, 65, 90, 97, 122, 48, 55, 125, 48, 55, 125, 48, 57, 65, -+ 70, 97, 102, 44, 125, 48, 57, 125, 48, 57, 125, 48, 57, 384, -+ 447, 384, 447, 384, 447, 41, 41, 80, 41, 41, 70, 41, 56, 41, -+ 121, 97, 109, 98, 105, 99, 101, 110, 105, 97, 110, 101, 115, 116, -+ 97, 110, 108, 109, 116, 105, 110, 101, 115, 101, 117, 109, 97, 107, -+ 110, 103, 97, 108, 105, 112, 111, 109, 111, 102, 111, 97, 104, 105, -+ 109, 105, 108, 108, 101, 103, 104, 105, 110, 101, 115, 101, 105, 100, -+ 110, 114, 97, 100, 105, 97, 110, 95, 65, 98, 111, 114, 105, 103, -+ 105, 110, 97, 108, 105, 97, 110, 97, 101, 109, 114, 111, 107, 101, -+ 101, 109, 111, 110, 116, 105, 99, 110, 101, 105, 102, 111, 114, 109, -+ 112, 114, 114, 105, 111, 116, 105, 108, 108, 105, 99, 115, 118, 101, -+ 114, 101, 116, 97, 110, 97, 103, 97, 114, 105, 121, 112, 116, 105, -+ 97, 110, 95, 72, 105, 101, 114, 111, 103, 108, 121, 112, 104, 115, -+ 104, 105, 111, 112, 105, 99, 111, 114, 103, 105, 97, 110, 97, 103, -+ 111, 108, 105, 116, 105, 99, 116, 104, 105, 99, 101, 101, 107, 106, -+ 114, 97, 114, 97, 116, 105, 109, 117, 107, 104, 105, 110, 117, 108, -+ 110, 111, 111, 98, 114, 101, 119, 114, 97, 103, 97, 110, 97, 112, -+ 101, 114, 105, 97, 108, 95, 65, 114, 97, 109, 97, 105, 99, 104, -+ 115, 101, 114, 105, 116, 101, 100, 99, 114, 105, 112, 116, 105, 111, -+ 110, 97, 108, 95, 80, 97, 104, 114, 108, 97, 118, 105, 116, 104, -+ 105, 97, 110, 118, 97, 110, 101, 115, 101, 105, 110, 116, 121, 116, -+ 104, 105, 110, 97, 100, 97, 97, 107, 97, 110, 97, 97, 104, 95, -+ 76, 105, 97, 109, 114, 111, 115, 104, 116, 104, 105, 101, 114, 111, -+ 116, 105, 110, 112, 99, 104, 97, 109, 110, 115, 98, 117, 101, 97, -+ 114, 95, 66, 117, 99, 100, 105, 97, 110, 105, 97, 110, 108, 110, -+ 97, 121, 97, 108, 97, 109, 100, 97, 105, 99, 116, 101, 105, 95, -+ 77, 97, 121, 101, 107, 110, 103, 111, 108, 105, 97, 110, 97, 110, -+ 109, 97, 114, 119, 95, 84, 97, 105, 95, 76, 117, 101, 111, 104, -+ 97, 109, 95, 100, 67, 104, 105, 107, 105, 95, 73, 80, 83, 84, -+ 116, 97, 108, 105, 99, 101, 114, 115, 105, 97, 110, 111, 117, 116, -+ 104, 95, 65, 114, 97, 98, 105, 97, 110, 117, 114, 107, 105, 99, -+ 105, 121, 97, 109, 97, 110, 121, 97, 97, 111, 103, 115, 95, 80, -+ 97, 101, 110, 105, 99, 105, 97, 110, 106, 97, 110, 103, 110, 105, -+ 99, 109, 117, 97, 114, 105, 116, 97, 110, 114, 97, 115, 104, 116, -+ 114, 97, 97, 118, 105, 97, 110, 110, 104, 97, 108, 97, 110, 100, -+ 97, 110, 101, 115, 101, 108, 114, 111, 116, 105, 95, 78, 97, 103, -+ 114, 105, 105, 97, 99, 103, 105, 109, 97, 98, 108, 111, 103, 97, -+ 110, 119, 97, 95, 76, 84, 86, 101, 104, 97, 109, 105, 101, 116, -+ 105, 108, 108, 117, 103, 117, 97, 97, 105, 110, 97, 98, 102, 101, -+ 116, 97, 110, 105, 110, 97, 103, 104, 97, 114, 105, 116, 105, 99, -+ 105, 110, 115, 112, 100, 123, 94, 125, 94, 46, 92, 93, 46, 92, -+ 93, 46, 92, 58, 92, 93, 94, 97, 98, 99, 100, 103, 108, 112, -+ 115, 117, 119, 120, 58, 92, 93, 58, 92, 93, 58, 92, 58, 92, -+ 93, 97, 98, 99, 100, 103, 108, 112, 115, 117, 119, 120, 58, 92, -+ 93, 108, 115, 58, 92, 93, 110, 112, 58, 92, 93, 117, 58, 92, -+ 93, 109, 58, 92, 93, 58, 92, 93, 58, 92, 93, 104, 58, 92, -+ 93, 97, 58, 92, 93, 58, 92, 93, 58, 92, 93, 99, 58, 92, -+ 93, 105, 58, 92, 93, 105, 58, 92, 93, 58, 92, 93, 58, 92, -+ 93, 108, 58, 92, 93, 97, 58, 92, 93, 110, 58, 92, 93, 107, -+ 58, 92, 93, 58, 92, 93, 58, 92, 93, 110, 58, 92, 93, 116, -+ 58, 92, 93, 114, 58, 92, 93, 108, 58, 92, 93, 58, 92, 93, -+ 58, 92, 93, 105, 58, 92, 93, 103, 58, 92, 93, 105, 58, 92, -+ 93, 116, 58, 92, 93, 58, 92, 93, 58, 92, 93, 114, 58, 92, -+ 93, 97, 58, 92, 93, 112, 58, 92, 93, 104, 58, 92, 93, 58, -+ 92, 93, 58, 92, 93, 111, 58, 92, 93, 119, 58, 92, 93, 101, -+ 58, 92, 93, 114, 58, 92, 93, 58, 92, 93, 58, 92, 93, 114, -+ 117, 58, 92, 93, 105, 58, 92, 93, 110, 58, 92, 93, 116, 58, -+ 92, 93, 58, 92, 93, 58, 92, 93, 110, 58, 92, 93, 99, 58, -+ 92, 93, 116, 58, 92, 93, 58, 92, 93, 58, 92, 93, 112, 58, -+ 92, 93, 97, 58, 92, 93, 99, 58, 92, 93, 101, 58, 92, 93, -+ 58, 92, 93, 58, 92, 93, 112, 58, 92, 93, 112, 58, 92, 93, -+ 101, 58, 92, 93, 114, 58, 92, 93, 58, 92, 93, 58, 92, 93, -+ 111, 58, 92, 93, 114, 58, 92, 93, 100, 58, 92, 93, 58, 92, -+ 93, 58, 92, 93, 100, 58, 92, 93, 105, 58, 92, 93, 103, 58, -+ 92, 93, 105, 58, 92, 93, 116, 58, 92, 93, 58, 92, 93, 58, -+ 92, 93, 108, 115, 58, 92, 93, 110, 112, 58, 92, 93, 117, 58, -+ 92, 93, 109, 58, 92, 93, 58, 92, 93, 58, 92, 93, 104, 58, -+ 92, 93, 97, 58, 92, 93, 58, 92, 93, 58, 92, 93, 99, 58, -+ 92, 93, 105, 58, 92, 93, 105, 58, 92, 93, 58, 92, 93, 58, -+ 92, 93, 108, 58, 92, 93, 97, 58, 92, 93, 110, 58, 92, 93, -+ 107, 58, 92, 93, 58, 92, 93, 58, 92, 93, 110, 58, 92, 93, -+ 116, 58, 92, 93, 114, 58, 92, 93, 108, 58, 92, 93, 58, 92, -+ 93, 58, 92, 93, 105, 58, 92, 93, 103, 58, 92, 93, 105, 58, -+ 92, 93, 116, 58, 92, 93, 58, 92, 93, 58, 92, 93, 114, 58, -+ 92, 93, 97, 58, 92, 93, 112, 58, 92, 93, 104, 58, 92, 93, -+ 58, 92, 93, 58, 92, 93, 111, 58, 92, 93, 119, 58, 92, 93, -+ 101, 58, 92, 93, 114, 58, 92, 93, 58, 92, 93, 58, 92, 93, -+ 114, 117, 58, 92, 93, 105, 58, 92, 93, 110, 58, 92, 93, 116, -+ 58, 92, 93, 58, 92, 93, 58, 92, 93, 110, 58, 92, 93, 99, -+ 58, 92, 93, 116, 58, 92, 93, 58, 92, 93, 58, 92, 93, 112, -+ 58, 92, 93, 97, 58, 92, 93, 99, 58, 92, 93, 101, 58, 92, -+ 93, 58, 92, 93, 58, 92, 93, 112, 58, 92, 93, 112, 58, 92, -+ 93, 101, 58, 92, 93, 114, 58, 92, 93, 58, 92, 93, 58, 92, -+ 93, 111, 58, 92, 93, 114, 58, 92, 93, 100, 58, 92, 93, 58, -+ 92, 93, 58, 92, 93, 100, 58, 92, 93, 105, 58, 92, 93, 103, -+ 58, 92, 93, 105, 58, 92, 93, 116, 58, 92, 93, 58, 92, 93, -+ 61, 92, 93, 61, 92, 93, 61, 92, 48, 55, 125, 48, 55, 125, -+ 48, 57, 65, 70, 97, 102, 384, 447, 384, 447, 384, 447, 384, 447, -+ 384, 447, 384, 447, 384, 447, 384, 447, 384, 447, 0, 32, 36, 40, -+ 41, 42, 43, 46, 63, 91, 92, 94, 123, 124, 1315, 1571, 1, 8, -+ 9, 13, 14, 34, 37, 255, 384, 447, 448, 479, 480, 495, 496, 503, -+ 504, 511, 42, 63, 95, 48, 57, 65, 90, 97, 122, 95, 48, 57, -+ 65, 90, 97, 122, 39, 48, 60, 63, 82, 95, 49, 55, 56, 57, -+ 65, 90, 97, 122, 48, 57, 105, 109, 115, 120, 48, 57, 41, 48, -+ 57, 33, 61, 95, 48, 57, 65, 90, 97, 122, 123, 41, 48, 57, -+ 60, 61, 62, 41, 45, 58, 105, 109, 115, 120, 43, 63, 43, 63, -+ 43, 63, 46, 58, 61, 48, 65, 66, 67, 68, 69, 71, 72, 75, -+ 76, 78, 80, 81, 82, 83, 85, 86, 87, 88, 90, 97, 98, 99, -+ 100, 101, 102, 103, 104, 107, 108, 110, 111, 112, 114, 115, 116, 117, -+ 118, 119, 120, 122, 49, 55, 56, 57, 48, 55, 48, 55, 48, 55, -+ 56, 57, 48, 55, 56, 57, 48, 57, 123, 39, 45, 60, 123, 48, -+ 57, 48, 57, 48, 57, 48, 57, 48, 57, 39, 60, 123, 123, 123, -+ 123, 48, 57, 65, 70, 97, 102, 48, 57, 65, 70, 97, 102, 48, -+ 57, 65, 70, 97, 102, 48, 57, 43, 63, 384, 447, 384, 447, 384, -+ 447, 41, 85, 41, 41, 67, 84, 65, 66, 67, 68, 69, 71, 72, -+ 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 88, -+ 89, 90, 110, 114, 118, 97, 101, 111, 114, 117, 97, 99, 102, 104, -+ 110, 111, 115, 117, 121, 109, 112, 101, 103, 116, 101, 108, 111, 114, -+ 117, 97, 101, 105, 103, 117, 109, 110, 97, 97, 104, 38, 97, 101, -+ 105, 108, 109, 111, 116, 117, 121, 97, 99, 101, 110, 111, 121, 101, -+ 100, 101, 107, 108, 111, 103, 108, 114, 115, 99, 100, 101, 102, 104, -+ 105, 111, 115, 101, 117, 97, 99, 104, 105, 107, 109, 111, 117, 121, -+ 97, 101, 104, 105, 103, 97, 97, 112, 115, 119, 105, 108, 112, 115, -+ 67, 76, 77, 78, 80, 83, 90, 45, 91, 92, 93, 0, 255, 384, -+ 447, 448, 479, 480, 495, 496, 503, 504, 511, 46, 58, 61, 48, 68, -+ 69, 72, 76, 78, 80, 81, 83, 85, 86, 87, 97, 98, 99, 100, -+ 101, 102, 103, 104, 108, 110, 111, 112, 114, 115, 116, 117, 118, 119, -+ 120, 49, 55, 56, 57, 65, 90, 105, 122, 48, 55, 48, 55, 48, -+ 55, 48, 55, 123, 123, 123, 123, 48, 57, 65, 70, 97, 102, 48, -+ 57, 65, 70, 97, 102, 48, 57, 65, 70, 97, 102, 384, 447, 384, -+ 447, 384, 447, 92, 1117, 1118, -128, 91, 95, 127, 861, 862, 69, 81, -+ 92, 0, 255, 384, 447, 448, 479, 480, 495, 496, 503, 504, 511, 69, -+ 384, 447, 384, 447, 384, 447, 92, 0, 255, 384, 447, 448, 479, 480, -+ 495, 496, 503, 504, 511, 69, 384, 447, 384, 447, 384, 447, 41, 10, -+ 0}; -+ -+static const char _regex_single_lengths[] = { -+ 0, 1, 20, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 3, 2, 3, 1, 2, 2, 1, 6, 2, 1, -+ 1, 1, 2, 1, 4, 7, 3, 3, 2, 3, 3, 2, 3, 3, 2, 1, 0, 1, 2, 0, 1, 1, 1, -+ 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 0, 1, 1, 2, 1, 1, 0, 0, 0, 1, 2, 1, 2, -+ 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, -+ 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, -+ 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 2, 15, 3, 3, 2, 14, 5, 5, 4, 4, -+ 3, 3, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, -+ 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 5, 4, 4, 4, 3, -+ 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, -+ 4, 4, 4, 4, 4, 3, 3, 5, 5, 4, 4, 3, 3, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, -+ 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, -+ 4, 4, 4, 4, 3, 3, 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, -+ 4, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 2, 0, 1, -+ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 1, 1, 6, 0, 4, 1, 3, 1, 1, 3, 7, -+ 2, 2, 2, 3, 41, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 3, 1, 1, 1, 0, 0, -+ 0, 2, 0, 0, 0, 2, 1, 3, 23, 3, 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, -+ 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, 3, 0, 7, 4, 3, 31, 0, 0, 0, 0, 1, 0, -+ 1, 1, 1, 0, 0, 0, 0, 0, 3, 2, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1}; -+ -+static const char _regex_range_lengths[] = { -+ 0, 0, 1, 3, 3, 3, 3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 3, 3, 3, 1, 0, 3, 1, 3, 3, -+ 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 1, 1, 0, 3, 3, 3, 3, -+ 3, 3, 3, 3, 3, 3, 1, 1, 3, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 0, 3, 3, -+ 4, 1, 1, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 1, 2, 2, 1, 0, 0, 1, 1, 1, 1, -+ 1, 0, 0, 0, 3, 3, 3, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 1, 1, 1, -+ 1, 0, 0, 0, 0, 3, 3, 3, 1, 1, 1, 3, 0, 6, 0, 1, 1, 1, 6, 0, 1, 1, 1, 0, 0}; -+ -+static const short _regex_index_offsets[] = { -+ 0, 0, 2, 24, 30, 36, 41, 47, 49, 55, 61, 67, -+ 72, 78, 82, 85, 92, 97, 103, 109, 112, 119, 125, 128, -+ 133, 138, 144, 149, 154, 162, 166, 170, 173, 177, 181, 184, -+ 188, 192, 195, 197, 199, 201, 207, 209, 212, 215, 217, 223, -+ 229, 235, 241, 246, 252, 257, 263, 268, 274, 276, 279, 284, -+ 288, 291, 294, 296, 298, 300, 302, 305, 307, 310, 313, 315, -+ 317, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, -+ 342, 344, 346, 350, 352, 354, 356, 358, 360, 362, 364, 366, -+ 368, 370, 372, 374, 376, 378, 380, 382, 384, 386, 388, 390, -+ 392, 395, 397, 399, 401, 403, 405, 408, 410, 412, 414, 416, -+ 418, 420, 422, 425, 427, 429, 431, 433, 435, 437, 439, 441, -+ 443, 445, 447, 449, 451, 453, 455, 457, 459, 461, 463, 466, -+ 468, 470, 472, 474, 476, 478, 480, 482, 484, 486, 488, 490, -+ 492, 494, 496, 498, 500, 502, 504, 507, 509, 511, 513, 515, -+ 517, 519, 521, 523, 525, 528, 530, 532, 534, 536, 538, 540, -+ 542, 544, 546, 548, 550, 552, 554, 556, 558, 560, 562, 564, -+ 566, 568, 570, 572, 574, 576, 578, 580, 582, 584, 586, 588, -+ 590, 592, 594, 596, 598, 600, 602, 604, 606, 608, 610, 612, -+ 614, 616, 618, 620, 622, 624, 626, 628, 630, 632, 634, 636, -+ 638, 640, 643, 645, 647, 649, 651, 653, 655, 657, 659, 661, -+ 663, 665, 667, 669, 671, 673, 675, 677, 679, 681, 683, 685, -+ 687, 689, 691, 693, 695, 697, 699, 701, 703, 705, 707, 709, -+ 711, 713, 715, 717, 719, 721, 723, 726, 728, 730, 732, 734, -+ 736, 738, 740, 742, 744, 746, 748, 750, 752, 754, 756, 758, -+ 760, 762, 764, 767, 769, 771, 773, 775, 777, 779, 781, 783, -+ 785, 787, 789, 791, 793, 795, 797, 802, 804, 806, 808, 810, -+ 812, 814, 816, 818, 820, 822, 824, 826, 828, 830, 832, 834, -+ 836, 839, 841, 843, 845, 847, 849, 851, 853, 855, 857, 860, -+ 862, 864, 866, 868, 870, 872, 876, 878, 880, 882, 884, 886, -+ 888, 890, 892, 895, 897, 899, 901, 903, 905, 907, 910, 912, -+ 914, 916, 918, 920, 922, 924, 926, 928, 930, 932, 934, 936, -+ 938, 940, 942, 944, 946, 948, 950, 952, 954, 956, 958, 960, -+ 962, 964, 966, 968, 970, 972, 974, 976, 978, 980, 982, 984, -+ 986, 988, 990, 992, 994, 996, 998, 1001, 1003, 1005, 1007, 1009, -+ 1011, 1013, 1018, 1020, 1022, 1024, 1026, 1028, 1030, 1032, 1034, 1036, -+ 1038, 1040, 1042, 1044, 1046, 1048, 1050, 1052, 1054, 1056, 1058, 1060, -+ 1062, 1064, 1066, 1068, 1070, 1072, 1074, 1076, 1078, 1080, 1082, 1084, -+ 1086, 1088, 1090, 1093, 1095, 1097, 1099, 1101, 1103, 1105, 1107, 1109, -+ 1111, 1113, 1115, 1117, 1119, 1121, 1123, 1125, 1127, 1129, 1131, 1134, -+ 1136, 1138, 1140, 1142, 1144, 1146, 1148, 1150, 1152, 1154, 1156, 1158, -+ 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, 1176, 1178, 1180, 1182, -+ 1184, 1186, 1188, 1190, 1192, 1194, 1197, 1199, 1201, 1203, 1205, 1207, -+ 1209, 1211, 1213, 1215, 1217, 1219, 1221, 1225, 1228, 1230, 1232, 1234, -+ 1236, 1238, 1240, 1242, 1244, 1248, 1250, 1252, 1254, 1256, 1258, 1260, -+ 1262, 1264, 1266, 1268, 1270, 1272, 1274, 1276, 1279, 1281, 1283, 1286, -+ 1288, 1290, 1292, 1294, 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, -+ 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, 1328, 1330, 1332, 1334, -+ 1338, 1342, 1345, 1361, 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, -+ 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, 1450, 1454, 1458, 1463, -+ 1468, 1473, 1478, 1482, 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, -+ 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, 1562, 1566, 1570, 1575, -+ 1580, 1585, 1590, 1594, 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, -+ 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, 1674, 1678, 1683, 1688, -+ 1693, 1698, 1702, 1706, 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, -+ 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, 1788, 1792, 1797, 1802, -+ 1806, 1810, 1815, 1820, 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, -+ 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, 1899, 1904, 1909, 1913, -+ 1917, 1922, 1927, 1932, 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, -+ 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, 2012, 2017, 2021, 2025, -+ 2030, 2035, 2040, 2045, 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, -+ 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, 2124, 2129, 2133, 2137, -+ 2141, 2145, 2148, 2150, 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, -+ 2172, 2174, 2176, 2202, 2205, 2210, 2215, 2226, 2228, 2234, 2237, 2244, -+ 2246, 2249, 2253, 2261, 2264, 2267, 2270, 2274, 2318, 2320, 2322, 2325, -+ 2328, 2330, 2332, 2333, 2339, 2341, 2343, 2345, 2347, 2351, 2353, 2355, -+ 2360, 2364, 2368, 2370, 2373, 2375, 2377, 2379, 2382, 2384, 2388, 2412, -+ 2416, 2422, 2432, 2435, 2437, 2440, 2446, 2450, 2453, 2456, 2458, 2461, -+ 2472, 2479, 2481, 2487, 2492, 2501, 2504, 2514, 2519, 2521, 2523, 2528, -+ 2530, 2534, 2535, 2543, 2554, 2558, 2594, 2596, 2598, 2600, 2602, 2604, -+ 2605, 2607, 2609, 2614, 2618, 2622, 2624, 2626, 2628, 2635, 2638, 2646, -+ 2648, 2650, 2652, 2654, 2662, 2664, 2666, 2668, 2670, 2672}; -+ -+static const short _regex_indicies[] = { -+ 0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, -+ 14, 15, 16, 17, 18, 19, 19, 19, 19, 20, 11, 2, -+ 22, 23, 23, 23, 23, 21, 24, 25, 25, 25, 25, 21, -+ 27, 27, 27, 27, 26, 28, 27, 27, 27, 27, 26, 29, -+ 26, 29, 30, 30, 30, 30, 26, 31, 30, 32, 30, 30, -+ 26, 29, 30, 32, 30, 30, 26, 33, 33, 33, 33, 26, -+ 28, 33, 33, 33, 33, 26, 34, 35, 36, 26, 37, 38, -+ 26, 39, 40, 30, 41, 30, 30, 26, 42, 42, 42, 42, -+ 26, 40, 42, 42, 42, 42, 26, 40, 30, 41, 30, 30, -+ 26, 43, 44, 21, 45, 46, 47, 47, 47, 47, 21, 24, -+ 48, 48, 48, 48, 21, 49, 50, 21, 48, 48, 48, 48, -+ 21, 51, 51, 51, 51, 21, 52, 51, 51, 51, 51, 21, -+ 23, 23, 23, 23, 21, 47, 47, 47, 47, 21, 45, 53, -+ 46, 54, 54, 54, 54, 21, 57, 58, 55, 56, 57, 58, -+ 59, 56, 57, 58, 56, 61, 62, 55, 60, 61, 62, 63, -+ 60, 61, 62, 60, 65, 66, 55, 64, 65, 66, 59, 64, -+ 65, 66, 64, 69, 68, 70, 67, 69, 71, 72, 74, 73, -+ 74, 74, 67, 75, 67, 77, 76, 67, 77, 78, 67, 77, -+ 67, 74, 80, 79, 74, 74, 67, 74, 80, 81, 74, 74, -+ 67, 74, 80, 74, 74, 74, 67, 74, 82, 74, 74, 74, -+ 67, 84, 84, 84, 84, 83, 85, 84, 84, 84, 84, 83, -+ 86, 86, 86, 86, 83, 87, 86, 86, 86, 86, 83, 88, -+ 88, 88, 88, 83, 88, 89, 88, 88, 88, 83, 91, 90, -+ 92, 91, 90, 95, 94, 94, 94, 93, 97, 99, 98, 96, -+ 101, 100, 96, 102, 100, 96, 104, 103, 105, 103, 106, 103, -+ 109, 108, 109, 110, 108, 111, 108, 109, 112, 108, 113, 114, -+ 108, 115, 108, 117, 116, 118, 119, 116, 120, 116, 121, 116, -+ 122, 116, 123, 116, 124, 116, 125, 116, 126, 116, 127, 116, -+ 128, 116, 129, 116, 130, 116, 131, 116, 132, 116, 133, 134, -+ 135, 116, 136, 116, 137, 116, 138, 116, 139, 116, 140, 116, -+ 141, 116, 142, 116, 143, 116, 144, 116, 145, 116, 146, 116, -+ 147, 116, 148, 116, 149, 116, 150, 116, 151, 116, 152, 116, -+ 153, 116, 154, 116, 155, 116, 156, 116, 157, 158, 116, 159, -+ 116, 160, 116, 161, 116, 162, 116, 163, 116, 164, 165, 116, -+ 166, 116, 167, 116, 168, 116, 169, 116, 170, 116, 171, 116, -+ 172, 116, 174, 175, 173, 176, 173, 177, 173, 178, 173, 179, -+ 173, 180, 173, 181, 173, 182, 173, 183, 173, 184, 173, 185, -+ 173, 186, 173, 187, 173, 188, 173, 189, 173, 190, 173, 191, -+ 173, 192, 173, 193, 173, 194, 173, 195, 196, 173, 197, 173, -+ 198, 173, 199, 173, 200, 173, 201, 173, 202, 173, 204, 203, -+ 205, 203, 206, 203, 207, 203, 208, 203, 209, 203, 210, 173, -+ 211, 173, 212, 173, 213, 173, 214, 173, 215, 173, 216, 173, -+ 217, 218, 173, 219, 173, 220, 173, 221, 173, 222, 173, 223, -+ 173, 224, 173, 225, 173, 226, 173, 227, 173, 228, 229, 116, -+ 230, 116, 231, 116, 232, 116, 233, 116, 234, 116, 235, 116, -+ 236, 116, 237, 116, 238, 116, 239, 116, 240, 116, 241, 116, -+ 242, 116, 243, 116, 244, 116, 245, 116, 246, 116, 247, 116, -+ 248, 116, 249, 116, 250, 116, 251, 116, 252, 116, 253, 116, -+ 254, 116, 255, 116, 256, 116, 257, 116, 258, 116, 259, 116, -+ 260, 116, 261, 116, 262, 116, 263, 116, 264, 116, 265, 116, -+ 266, 116, 267, 116, 268, 116, 269, 116, 270, 116, 271, 116, -+ 272, 116, 273, 116, 274, 116, 275, 116, 276, 116, 277, 116, -+ 278, 116, 279, 116, 280, 116, 281, 116, 282, 116, 283, 116, -+ 284, 116, 285, 116, 286, 287, 116, 288, 116, 289, 116, 290, -+ 116, 291, 116, 292, 116, 293, 116, 294, 116, 295, 116, 296, -+ 116, 297, 116, 298, 116, 300, 299, 301, 299, 302, 299, 303, -+ 299, 304, 299, 305, 116, 306, 116, 307, 116, 308, 116, 309, -+ 116, 310, 116, 311, 116, 312, 116, 313, 116, 314, 116, 315, -+ 116, 316, 116, 317, 116, 318, 116, 319, 116, 320, 116, 321, -+ 116, 322, 116, 323, 116, 324, 116, 325, 116, 326, 116, 327, -+ 116, 328, 116, 329, 330, 116, 331, 116, 332, 116, 333, 116, -+ 334, 116, 335, 116, 336, 116, 337, 116, 338, 116, 339, 116, -+ 340, 116, 341, 116, 342, 116, 343, 116, 344, 116, 345, 116, -+ 346, 116, 347, 116, 348, 116, 349, 116, 350, 351, 116, 352, -+ 116, 353, 116, 354, 116, 355, 116, 356, 116, 357, 116, 358, -+ 116, 359, 116, 360, 116, 361, 116, 362, 116, 363, 116, 364, -+ 116, 365, 116, 366, 116, 367, 368, 369, 370, 116, 371, 116, -+ 372, 116, 373, 116, 374, 116, 375, 116, 376, 116, 377, 116, -+ 378, 116, 379, 116, 380, 116, 381, 116, 382, 116, 383, 116, -+ 384, 116, 385, 116, 386, 116, 387, 116, 388, 389, 116, 390, -+ 116, 391, 116, 392, 116, 393, 116, 394, 116, 395, 116, 396, -+ 116, 397, 116, 398, 116, 400, 401, 399, 402, 399, 403, 399, -+ 404, 399, 405, 399, 406, 399, 407, 399, 408, 409, 410, 399, -+ 411, 399, 412, 399, 413, 399, 414, 399, 415, 399, 416, 399, -+ 417, 399, 418, 399, 419, 420, 399, 421, 399, 422, 399, 423, -+ 399, 424, 399, 425, 399, 426, 399, 428, 429, 427, 430, 427, -+ 431, 427, 432, 427, 433, 427, 434, 427, 435, 427, 436, 427, -+ 437, 427, 438, 427, 439, 427, 441, 440, 442, 440, 443, 440, -+ 444, 440, 445, 440, 446, 440, 447, 440, 448, 440, 449, 440, -+ 450, 427, 451, 427, 452, 427, 453, 427, 454, 427, 455, 427, -+ 456, 427, 457, 427, 458, 427, 459, 427, 460, 427, 461, 427, -+ 463, 462, 464, 462, 465, 462, 466, 462, 467, 462, 468, 462, -+ 469, 462, 470, 462, 471, 462, 472, 462, 473, 116, 474, 116, -+ 475, 116, 476, 477, 116, 478, 116, 479, 116, 480, 116, 481, -+ 116, 482, 116, 483, 116, 484, 485, 486, 487, 116, 488, 116, -+ 489, 116, 490, 116, 491, 116, 492, 116, 493, 116, 494, 116, -+ 495, 116, 496, 116, 497, 116, 498, 116, 499, 116, 500, 116, -+ 501, 116, 502, 116, 503, 116, 504, 116, 505, 116, 506, 116, -+ 507, 116, 508, 116, 509, 116, 510, 116, 511, 116, 512, 116, -+ 513, 116, 514, 116, 515, 116, 516, 116, 517, 116, 518, 116, -+ 519, 116, 520, 116, 521, 116, 522, 116, 523, 116, 525, 526, -+ 524, 527, 524, 528, 524, 529, 524, 530, 524, 531, 524, 532, -+ 524, 533, 524, 534, 524, 535, 524, 536, 524, 537, 524, 538, -+ 524, 539, 116, 540, 116, 541, 116, 542, 116, 543, 116, 544, -+ 116, 545, 116, 547, 548, 546, 549, 546, 550, 546, 551, 546, -+ 552, 546, 553, 546, 554, 546, 555, 546, 556, 546, 557, 546, -+ 558, 546, 559, 546, 560, 546, 561, 546, 562, 546, 563, 546, -+ 564, 546, 565, 546, 566, 546, 567, 546, 568, 546, 569, 546, -+ 570, 546, 571, 546, 572, 546, 573, 546, 574, 546, 575, 546, -+ 576, 546, 577, 546, 578, 546, 579, 580, 546, 581, 546, 582, -+ 546, 583, 546, 584, 546, 585, 546, 586, 546, 587, 546, 588, -+ 546, 589, 546, 590, 546, 591, 546, 592, 546, 593, 594, 595, -+ 116, 596, 597, 116, 598, 116, 599, 116, 600, 116, 601, 116, -+ 602, 116, 603, 116, 604, 116, 605, 116, 606, 607, 608, 116, -+ 609, 116, 610, 116, 611, 116, 612, 116, 613, 116, 614, 116, -+ 615, 116, 616, 116, 617, 116, 618, 116, 619, 116, 620, 116, -+ 621, 116, 622, 116, 623, 624, 116, 625, 116, 626, 116, 627, -+ 628, 116, 629, 116, 630, 116, 631, 116, 632, 116, 633, 116, -+ 634, 116, 635, 116, 636, 116, 637, 116, 638, 116, 639, 116, -+ 640, 116, 641, 116, 642, 116, 643, 116, 644, 116, 645, 116, -+ 646, 116, 647, 116, 648, 116, 650, 649, 652, 651, 653, 649, -+ 649, 651, 656, 657, 654, 655, 656, 657, 658, 655, 656, 657, -+ 655, 660, 661, 654, 662, 663, 664, 665, 666, 667, 668, 669, -+ 670, 671, 672, 673, 659, 660, 661, 654, 659, 660, 661, 674, -+ 659, 660, 661, 659, 660, 661, 654, 675, 676, 677, 678, 679, -+ 680, 681, 682, 683, 684, 685, 659, 660, 661, 654, 686, 687, -+ 659, 660, 661, 654, 688, 689, 659, 660, 661, 654, 690, 659, -+ 660, 661, 654, 691, 659, 692, 661, 654, 659, 660, 661, 693, -+ 659, 660, 661, 654, 694, 659, 660, 661, 654, 695, 659, 696, -+ 661, 654, 659, 660, 661, 697, 659, 660, 661, 654, 698, 659, -+ 660, 661, 654, 699, 659, 660, 661, 654, 700, 659, 701, 661, -+ 654, 659, 660, 661, 702, 659, 660, 661, 654, 703, 659, 660, -+ 661, 654, 704, 659, 660, 661, 654, 705, 659, 660, 661, 654, -+ 706, 659, 707, 661, 654, 659, 660, 661, 708, 659, 660, 661, -+ 654, 709, 659, 660, 661, 654, 710, 659, 660, 661, 654, 711, -+ 659, 660, 661, 654, 712, 659, 713, 661, 654, 659, 660, 661, -+ 714, 659, 660, 661, 654, 715, 659, 660, 661, 654, 716, 659, -+ 660, 661, 654, 717, 659, 660, 661, 654, 718, 659, 719, 661, -+ 654, 659, 660, 661, 720, 659, 660, 661, 654, 721, 659, 660, -+ 661, 654, 722, 659, 660, 661, 654, 723, 659, 660, 661, 654, -+ 724, 659, 725, 661, 654, 659, 660, 661, 726, 659, 660, 661, -+ 654, 727, 659, 660, 661, 654, 728, 659, 660, 661, 654, 729, -+ 659, 660, 661, 654, 730, 659, 731, 661, 654, 659, 660, 661, -+ 732, 659, 660, 661, 654, 733, 734, 659, 660, 661, 654, 735, -+ 659, 660, 661, 654, 736, 659, 660, 661, 654, 737, 659, 738, -+ 661, 654, 659, 660, 661, 739, 659, 660, 661, 654, 740, 659, -+ 660, 661, 654, 741, 659, 660, 661, 654, 742, 659, 743, 661, -+ 654, 659, 660, 661, 744, 659, 660, 661, 654, 745, 659, 660, -+ 661, 654, 746, 659, 660, 661, 654, 747, 659, 660, 661, 654, -+ 748, 659, 749, 661, 654, 659, 660, 661, 750, 659, 660, 661, -+ 654, 751, 659, 660, 661, 654, 752, 659, 660, 661, 654, 753, -+ 659, 660, 661, 654, 754, 659, 755, 661, 654, 659, 660, 661, -+ 756, 659, 660, 661, 654, 757, 659, 660, 661, 654, 758, 659, -+ 660, 661, 654, 759, 659, 760, 661, 654, 659, 660, 661, 761, -+ 659, 660, 661, 654, 762, 659, 660, 661, 654, 763, 659, 660, -+ 661, 654, 764, 659, 660, 661, 654, 765, 659, 660, 661, 654, -+ 766, 659, 767, 661, 654, 659, 660, 661, 768, 659, 660, 661, -+ 654, 769, 770, 659, 660, 661, 654, 771, 772, 659, 660, 661, -+ 654, 773, 659, 660, 661, 654, 774, 659, 775, 661, 654, 659, -+ 660, 661, 776, 659, 660, 661, 654, 777, 659, 660, 661, 654, -+ 778, 659, 779, 661, 654, 659, 660, 661, 780, 659, 660, 661, -+ 654, 781, 659, 660, 661, 654, 782, 659, 660, 661, 654, 783, -+ 659, 784, 661, 654, 659, 660, 661, 785, 659, 660, 661, 654, -+ 786, 659, 660, 661, 654, 787, 659, 660, 661, 654, 788, 659, -+ 660, 661, 654, 789, 659, 790, 661, 654, 659, 660, 661, 791, -+ 659, 660, 661, 654, 792, 659, 660, 661, 654, 793, 659, 660, -+ 661, 654, 794, 659, 660, 661, 654, 795, 659, 796, 661, 654, -+ 659, 660, 661, 797, 659, 660, 661, 654, 798, 659, 660, 661, -+ 654, 799, 659, 660, 661, 654, 800, 659, 660, 661, 654, 801, -+ 659, 802, 661, 654, 659, 660, 661, 803, 659, 660, 661, 654, -+ 804, 659, 660, 661, 654, 805, 659, 660, 661, 654, 806, 659, -+ 660, 661, 654, 807, 659, 808, 661, 654, 659, 660, 661, 809, -+ 659, 660, 661, 654, 810, 659, 660, 661, 654, 811, 659, 660, -+ 661, 654, 812, 659, 660, 661, 654, 813, 659, 814, 661, 654, -+ 659, 660, 661, 815, 659, 660, 661, 654, 816, 817, 659, 660, -+ 661, 654, 818, 659, 660, 661, 654, 819, 659, 660, 661, 654, -+ 820, 659, 821, 661, 654, 659, 660, 661, 822, 659, 660, 661, -+ 654, 823, 659, 660, 661, 654, 824, 659, 660, 661, 654, 825, -+ 659, 826, 661, 654, 659, 660, 661, 827, 659, 660, 661, 654, -+ 828, 659, 660, 661, 654, 829, 659, 660, 661, 654, 830, 659, -+ 660, 661, 654, 831, 659, 832, 661, 654, 659, 660, 661, 833, -+ 659, 660, 661, 654, 834, 659, 660, 661, 654, 835, 659, 660, -+ 661, 654, 836, 659, 660, 661, 654, 837, 659, 838, 661, 654, -+ 659, 660, 661, 839, 659, 660, 661, 654, 840, 659, 660, 661, -+ 654, 841, 659, 660, 661, 654, 842, 659, 843, 661, 654, 659, -+ 660, 661, 844, 659, 660, 661, 654, 845, 659, 660, 661, 654, -+ 846, 659, 660, 661, 654, 847, 659, 660, 661, 654, 848, 659, -+ 660, 661, 654, 849, 659, 850, 661, 654, 659, 660, 661, 851, -+ 659, 853, 854, 654, 852, 853, 854, 658, 852, 853, 854, 852, -+ 856, 855, 857, 856, 855, 860, 859, 859, 859, 858, 862, 861, -+ 863, 861, 864, 861, 866, 865, 867, 865, 868, 865, 870, 869, -+ 871, 869, 872, 869, 873, 876, 877, 878, 879, 880, 881, 882, -+ 883, 884, 885, 886, 887, 888, 875, 893, 875, 876, 875, 875, -+ 889, 890, 891, 892, 889, 874, 895, 896, 894, 23, 23, 23, -+ 23, 897, 25, 25, 25, 25, 897, 899, 30, 902, 903, 904, -+ 30, 900, 901, 30, 30, 898, 44, 897, 47, 47, 47, 47, -+ 44, 897, 43, 44, 897, 905, 906, 48, 48, 48, 48, 897, -+ 907, 897, 49, 50, 897, 908, 909, 910, 897, 45, 53, 46, -+ 54, 54, 54, 54, 897, 912, 913, 911, 915, 916, 914, 918, -+ 919, 917, 56, 60, 64, 920, 923, 926, 927, 928, 929, 930, -+ 931, 932, 933, 934, 934, 935, 936, 937, 938, 934, 939, 940, -+ 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 934, -+ 952, 953, 954, 955, 956, 957, 934, 958, 959, 960, 961, 924, -+ 925, 922, 963, 962, 964, 962, 966, 967, 965, 969, 967, 968, -+ 967, 970, 973, 972, 975, 68, 977, 71, 979, 978, 976, 981, -+ 980, 982, 980, 984, 983, 985, 983, 987, 988, 989, 986, 991, -+ 990, 994, 993, 999, 996, 997, 998, 995, 1000, 1001, 1002, 995, -+ 94, 94, 94, 1003, 98, 1004, 1006, 1007, 1005, 1009, 1008, 1010, -+ 1008, 1011, 1008, 1013, 1014, 1012, 109, 108, 109, 1016, 1017, 108, -+ 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, -+ 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1018, -+ 1043, 1044, 1045, 1042, 1046, 1047, 1048, 1049, 1050, 1042, 1052, 1053, -+ 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1051, 1062, 1063, 1061, 1064, -+ 1042, 1065, 1066, 1042, 1067, 1068, 1069, 1070, 1071, 1042, 1072, 1073, -+ 1074, 1042, 1076, 1077, 1075, 1078, 1079, 1042, 1080, 1042, 1081, 1082, -+ 1042, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1083, -+ 1095, 1096, 1097, 1098, 1099, 1100, 1094, 1102, 1101, 1104, 1105, 1106, -+ 1107, 1108, 1103, 1109, 1110, 1111, 1112, 1042, 1114, 1115, 1116, 1117, -+ 1118, 1119, 1120, 1121, 1113, 1122, 1123, 1042, 1125, 1126, 1127, 1128, -+ 1129, 1130, 1131, 1132, 1133, 1124, 1134, 1135, 1136, 1137, 1042, 1138, -+ 1042, 1139, 1042, 1140, 1141, 1142, 1143, 1042, 1144, 1042, 1146, 1147, -+ 1148, 1145, 649, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1149, 1158, -+ 1159, 1160, 1161, 1157, 1162, 1163, 1164, 1165, 1162, 874, 655, 1167, -+ 852, 1166, 1169, 1173, 1174, 1175, 1176, 1176, 1177, 1178, 1179, 1176, -+ 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1176, 1190, -+ 1191, 1192, 1193, 1194, 1195, 1176, 1196, 1197, 1198, 1170, 1171, 1172, -+ 1172, 1168, 1200, 1199, 1201, 1199, 1203, 1202, 1204, 1202, 1207, 1206, -+ 1209, 1211, 1210, 1214, 1213, 1219, 1216, 1217, 1218, 1215, 1220, 1221, -+ 1222, 1215, 859, 859, 859, 1223, 1225, 1224, 1226, 1224, 1227, 1224, -+ 1229, 1230, 1231, 1228, 1228, 1228, 874, 1233, 1234, 1232, 1236, 1235, -+ 1237, 1238, 1239, 1240, 1237, 874, 1242, 1241, 1244, 1243, 1245, 1243, -+ 1246, 1243, 1248, 1247, 1249, 1250, 1251, 1252, 1249, 874, 1254, 1253, -+ 1256, 1255, 1257, 1255, 1258, 1255, 1260, 1259, 1262, 1261, 0}; -+ -+static const short _regex_trans_targs[] = { -+ 746, 746, 746, 746, 746, 748, 749, 750, 746, 751, 752, 753, 746, 754, 746, -+ 746, 755, 756, 757, 758, 746, 746, 746, 3, 746, 4, 746, 6, 7, 746, -+ 8, 746, 9, 12, 746, 14, 746, 746, 746, 16, 746, 18, 17, 746, 19, -+ 746, 746, 20, 21, 746, 22, 25, 746, 27, 28, 746, 29, 30, 31, 746, -+ 32, 33, 34, 746, 35, 36, 37, 746, 38, 746, 772, 40, 42, 46, 49, -+ 43, 44, 746, 45, 47, 746, 48, 746, 746, 51, 746, 53, 746, 55, 746, -+ 746, 57, 746, 746, 58, 746, 746, 60, 59, 783, 61, 783, 783, 746, 746, -+ 64, 746, 787, 65, 787, 67, 787, 69, 787, 70, 787, 790, 790, 73, 76, -+ 74, 75, 790, 77, 78, 79, 80, 790, 82, 83, 84, 85, 790, 87, 92, -+ 94, 88, 89, 90, 91, 790, 93, 790, 95, 790, 97, 98, 99, 100, 790, -+ 102, 103, 104, 105, 106, 790, 108, 109, 111, 110, 790, 112, 113, 790, 115, -+ 120, 116, 117, 118, 119, 790, 121, 790, 790, 123, 139, 124, 125, 126, 127, -+ 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 790, 140, 141, 790, -+ 143, 144, 790, 145, 146, 147, 148, 790, 790, 150, 151, 790, 153, 154, 790, -+ 156, 157, 158, 159, 160, 161, 790, 163, 167, 164, 165, 166, 790, 168, 169, -+ 170, 171, 790, 173, 177, 174, 175, 176, 790, 178, 179, 180, 181, 182, 183, -+ 790, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, -+ 199, 200, 201, 790, 203, 204, 205, 206, 207, 790, 209, 210, 211, 212, 213, -+ 790, 215, 216, 217, 218, 219, 220, 221, 790, 223, 224, 225, 790, 227, 228, -+ 790, 230, 235, 231, 232, 233, 234, 790, 236, 237, 238, 239, 790, 799, 790, -+ 242, 790, 244, 245, 790, 247, 248, 249, 790, 251, 252, 253, 254, 255, 790, -+ 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 790, 271, -+ 277, 272, 273, 274, 275, 276, 790, 278, 279, 280, 281, 282, 283, 284, 285, -+ 286, 287, 288, 289, 290, 291, 295, 292, 293, 294, 790, 296, 297, 298, 299, -+ 790, 301, 302, 303, 304, 305, 790, 307, 310, 314, 319, 308, 309, 790, 311, -+ 312, 313, 790, 315, 316, 317, 318, 790, 320, 321, 322, 323, 790, 325, 332, -+ 326, 327, 328, 329, 330, 331, 790, 333, 790, 790, 790, 335, 336, 790, 338, -+ 339, 340, 790, 342, 344, 349, 343, 790, 345, 346, 347, 348, 790, 790, 351, -+ 354, 352, 353, 790, 355, 356, 790, 790, 358, 364, 359, 360, 361, 362, 363, -+ 790, 365, 366, 367, 790, 790, 369, 370, 371, 372, 373, 374, 375, 376, 790, -+ 378, 379, 380, 381, 382, 383, 790, 385, 386, 387, 388, 790, 790, 390, 391, -+ 392, 393, 394, 395, 396, 397, 790, 790, 400, 401, 790, 403, 408, 404, 405, -+ 406, 407, 790, 409, 410, 415, 421, 433, 411, 412, 413, 414, 790, 416, 417, -+ 418, 419, 420, 790, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, -+ 790, 434, 435, 436, 437, 790, 439, 440, 790, 442, 443, 444, 445, 790, 790, -+ 447, 452, 448, 449, 450, 451, 790, 453, 454, 455, 456, 457, 458, 790, 460, -+ 461, 462, 790, 464, 465, 790, 790, 467, 473, 468, 469, 470, 471, 472, 790, -+ 474, 475, 476, 477, 478, 479, 790, 481, 482, 483, 484, 790, 486, 487, 488, -+ 489, 790, 491, 492, 493, 494, 495, 496, 790, 498, 507, 499, 500, 501, 502, -+ 503, 504, 505, 506, 790, 508, 509, 790, 511, 519, 528, 512, 515, 513, 514, -+ 790, 516, 517, 518, 790, 520, 521, 522, 525, 790, 523, 524, 790, 526, 527, -+ 790, 529, 790, 531, 532, 533, 790, 535, 536, 790, 537, 790, 539, 543, 540, -+ 541, 542, 790, 544, 545, 546, 547, 790, 549, 550, 551, 552, 553, 790, 790, -+ 790, 790, 790, 790, 0, 560, 561, 562, 817, 819, 563, 564, 565, 819, 567, -+ 568, 569, 570, 651, 666, 672, 678, 684, 690, 696, 707, 713, 719, 724, 819, -+ 571, 586, 592, 598, 604, 610, 616, 627, 633, 639, 644, 572, 581, 573, 577, -+ 574, 575, 576, 819, 578, 579, 580, 819, 582, 583, 584, 585, 819, 587, 588, -+ 589, 590, 591, 819, 593, 594, 595, 596, 597, 819, 599, 600, 601, 602, 603, -+ 819, 605, 606, 607, 608, 609, 819, 611, 612, 613, 614, 615, 819, 617, 622, -+ 618, 619, 620, 621, 819, 623, 624, 625, 626, 819, 628, 629, 630, 631, 632, -+ 819, 634, 635, 636, 637, 638, 819, 640, 641, 642, 643, 819, 645, 646, 647, -+ 648, 649, 650, 819, 652, 661, 653, 657, 654, 655, 656, 819, 658, 659, 660, -+ 819, 662, 663, 664, 665, 819, 667, 668, 669, 670, 671, 819, 673, 674, 675, -+ 676, 677, 819, 679, 680, 681, 682, 683, 819, 685, 686, 687, 688, 689, 819, -+ 691, 692, 693, 694, 695, 819, 697, 702, 698, 699, 700, 701, 819, 703, 704, -+ 705, 706, 819, 708, 709, 710, 711, 712, 819, 714, 715, 716, 717, 718, 819, -+ 720, 721, 722, 723, 819, 725, 726, 727, 728, 729, 730, 819, 731, 732, 733, -+ 819, 735, 819, 819, 736, 819, 819, 819, 739, 819, 838, 838, 742, 838, 843, -+ 843, 745, 843, 746, 0, 746, 746, 746, 747, 746, 759, 760, 746, 761, 762, -+ 763, 746, 782, 746, 746, 784, 785, 786, 746, 746, 1, 2, 746, 746, 5, -+ 9, 10, 11, 13, 15, 746, 746, 746, 23, 24, 26, 746, 746, 746, 746, -+ 746, 746, 746, 746, 746, 746, 746, 746, 764, 766, 768, 746, 746, 746, 746, -+ 746, 746, 746, 746, 746, 769, 746, 746, 746, 746, 746, 746, 746, 746, 746, -+ 770, 746, 746, 746, 771, 746, 776, 746, 777, 778, 746, 746, 746, 746, 746, -+ 779, 746, 746, 765, 746, 746, 767, 768, 746, 768, 746, 746, 746, 746, 746, -+ 746, 746, 39, 774, 41, 746, 773, 746, 746, 775, 746, 746, 50, 52, 54, -+ 746, 56, 746, 746, 746, 746, 780, 780, 780, 781, 746, 746, 746, 746, 746, -+ 746, 746, 746, 746, 746, 62, 63, 788, 787, 789, 787, 66, 68, 790, 791, -+ 792, 793, 795, 796, 797, 798, 800, 801, 802, 803, 804, 806, 807, 808, 809, -+ 810, 811, 812, 813, 814, 815, 816, 790, 71, 72, 81, 86, 96, 101, 107, -+ 114, 790, 122, 790, 790, 142, 790, 794, 790, 155, 162, 790, 149, 152, 172, -+ 184, 202, 208, 214, 222, 226, 229, 240, 246, 250, 790, 241, 243, 256, 270, -+ 300, 306, 324, 790, 790, 334, 337, 341, 790, 790, 790, 790, 790, 350, 790, -+ 357, 790, 805, 790, 377, 384, 790, 368, 790, 790, 389, 398, 790, 790, 399, -+ 402, 438, 441, 790, 790, 790, 790, 790, 446, 790, 790, 790, 459, 463, 790, -+ 466, 790, 480, 485, 790, 790, 790, 490, 497, 510, 530, 534, 538, 548, 554, -+ 555, 556, 557, 558, 790, 790, 790, 790, 790, 818, 818, 818, 818, 818, 818, -+ 818, 818, 819, 819, 820, 821, 819, 819, 833, 834, 835, 819, 566, 819, 822, -+ 824, 819, 819, 819, 819, 819, 819, 826, 819, 819, 819, 819, 819, 819, 827, -+ 819, 819, 819, 819, 819, 819, 828, 829, 819, 819, 819, 819, 819, 830, 819, -+ 823, 819, 819, 825, 819, 819, 819, 819, 819, 819, 819, 734, 819, 819, 819, -+ 819, 831, 831, 831, 832, 819, 819, 819, 819, 819, 819, 737, 738, 836, 837, -+ 836, 836, 836, 836, 836, 838, 839, 838, 840, 841, 842, 838, 838, 838, 838, -+ 740, 741, 843, 844, 843, 845, 846, 847, 843, 843, 843, 843, 743, 744, 848, -+ 848, 849, 849}; -+ -+static const short _regex_trans_actions[] = { -+ 827, 631, 765, 731, 723, 45, 903, 903, 897, 45, 912, 45, 900, 903, 729, -+ 741, 0, 45, 45, 923, 737, 841, 747, 0, 743, 3, 839, 3, 0, 761, -+ 3, 759, 870, 3, 751, 0, 749, 755, 753, 0, 757, 3, 0, 745, 0, -+ 725, 727, 27, 3, 763, 0, 3, 673, 0, 25, 829, 0, 0, 0, 603, -+ 0, 0, 0, 601, 0, 0, 0, 831, 0, 675, 17, 0, 7, 870, 3, -+ 17, 17, 663, 17, 870, 661, 870, 665, 837, 3, 671, 3, 669, 3, 667, -+ 833, 0, 677, 835, 0, 679, 845, 0, 11, 29, 13, 31, 0, 843, 769, -+ 0, 771, 59, 0, 53, 0, 51, 0, 49, 0, 47, 359, 315, 0, 0, -+ 0, 0, 127, 0, 0, 0, 0, 129, 0, 0, 0, 0, 131, 0, 0, -+ 0, 0, 0, 0, 0, 133, 0, 135, 0, 137, 0, 0, 0, 0, 139, -+ 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 143, 0, 0, 145, 0, -+ 0, 0, 0, 0, 0, 147, 0, 149, 341, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 153, -+ 0, 0, 155, 0, 0, 0, 0, 157, 343, 0, 0, 159, 0, 0, 161, -+ 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 165, 0, 0, -+ 0, 0, 167, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, -+ 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 173, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, -+ 177, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 181, 0, 0, -+ 183, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 187, 45, 357, -+ 0, 189, 0, 0, 191, 0, 0, 0, 193, 0, 0, 0, 0, 0, 195, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, -+ 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, -+ 203, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 207, 0, -+ 0, 0, 209, 0, 0, 0, 0, 211, 0, 0, 0, 0, 213, 0, 0, -+ 0, 0, 0, 0, 0, 0, 215, 0, 217, 345, 219, 0, 0, 221, 0, -+ 0, 0, 223, 0, 0, 0, 0, 225, 0, 0, 0, 0, 227, 229, 0, -+ 0, 0, 0, 231, 0, 0, 233, 347, 0, 0, 0, 0, 0, 0, 0, -+ 235, 0, 0, 0, 237, 349, 0, 0, 0, 0, 0, 0, 0, 0, 239, -+ 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 243, 351, 0, 0, -+ 0, 0, 0, 0, 0, 0, 245, 247, 0, 0, 249, 0, 0, 0, 0, -+ 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, -+ 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 257, 0, 0, 0, 0, 259, 0, 0, 261, 0, 0, 0, 0, 263, 353, -+ 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 267, 0, -+ 0, 0, 269, 0, 0, 271, 355, 0, 0, 0, 0, 0, 0, 0, 273, -+ 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 277, 0, 0, 0, -+ 0, 279, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 283, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, -+ 287, 0, 0, 0, 289, 0, 0, 0, 0, 291, 0, 0, 293, 0, 0, -+ 295, 0, 297, 0, 0, 0, 299, 0, 0, 303, 0, 301, 0, 0, 0, -+ 0, 0, 305, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 309, 311, -+ 119, 121, 123, 125, 39, 0, 35, 33, 37, 539, 0, 0, 0, 377, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 381, 0, 0, 0, 385, 0, 0, 0, 0, 389, 0, 0, -+ 0, 0, 0, 393, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, -+ 401, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 409, 0, 0, -+ 0, 0, 0, 0, 413, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, -+ 421, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 429, 0, 0, 0, -+ 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, -+ 383, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 391, 0, 0, 0, -+ 0, 0, 395, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 403, -+ 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 411, 0, 0, -+ 0, 0, 415, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 423, -+ 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, -+ 533, 0, 471, 535, 0, 475, 537, 503, 0, 505, 569, 557, 0, 559, 587, -+ 575, 0, 577, 633, 0, 777, 775, 637, 45, 597, 0, 0, 609, 0, 45, -+ 0, 635, 909, 599, 773, 0, 45, 45, 629, 779, 0, 0, 821, 819, 1, -+ 855, 855, 1, 0, 3, 735, 733, 739, 1, 1, 0, 783, 615, 613, 785, -+ 619, 617, 787, 623, 621, 781, 817, 721, 5, 852, 915, 639, 647, 611, 695, -+ 607, 717, 699, 715, 683, 0, 605, 713, 691, 703, 687, 719, 641, 657, 645, -+ 0, 693, 659, 655, 906, 697, 45, 651, 45, 0, 653, 689, 649, 701, 685, -+ 7, 643, 791, 15, 867, 795, 858, 919, 793, 927, 847, 811, 711, 709, 809, -+ 681, 801, 7, 17, 849, 799, 17, 876, 797, 17, 873, 815, 1, 1, 1, -+ 803, 0, 813, 707, 705, 805, 19, 23, 21, 45, 882, 894, 888, 807, 825, -+ 789, 627, 625, 823, 767, 0, 0, 45, 55, 45, 57, 0, 0, 317, 45, -+ 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, -+ 45, 45, 45, 45, 45, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, -+ 0, 319, 0, 61, 63, 0, 65, 45, 67, 0, 0, 321, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, -+ 0, 0, 0, 323, 79, 0, 0, 0, 69, 71, 73, 75, 77, 0, 325, -+ 0, 81, 45, 83, 0, 0, 327, 0, 329, 85, 0, 0, 87, 89, 0, -+ 0, 0, 0, 331, 91, 93, 95, 97, 0, 99, 101, 103, 0, 0, 333, -+ 0, 105, 0, 0, 107, 109, 111, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 313, 335, 113, 115, 117, 375, 361, 363, 365, 367, 369, -+ 371, 373, 509, 491, 45, 0, 511, 507, 0, 45, 45, 531, 0, 499, 5, -+ 9, 473, 497, 489, 439, 457, 493, 0, 437, 485, 461, 481, 451, 441, 0, -+ 487, 453, 449, 495, 455, 445, 45, 0, 447, 483, 443, 459, 479, 7, 517, -+ 15, 861, 519, 15, 864, 513, 469, 467, 527, 477, 521, 0, 515, 465, 463, -+ 523, 19, 23, 21, 45, 879, 891, 885, 525, 529, 501, 0, 0, 549, 0, -+ 543, 541, 551, 547, 545, 563, 0, 561, 0, 45, 45, 567, 553, 565, 555, -+ 0, 0, 581, 0, 579, 0, 45, 45, 585, 571, 583, 573, 0, 0, 591, -+ 589, 595, 593}; -+ -+static const short _regex_to_state_actions[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 41, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 41, 0, 41, 0, 0, 0, 0, 41, 0, 0, 0, 0, 41, 41}; -+ -+static const short _regex_from_state_actions[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 43, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 43, 0, 0, 0, 0, 43, 0, 0, 0, 0, 43, 43}; -+ -+static const short _regex_eof_actions[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 39, 39, 39, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -+ -+static const short _regex_eof_trans[] = { -+ 0, 1, 1, 22, 22, 27, 27, 27, 27, 27, 27, 27, -+ 27, 27, 27, 27, 27, 27, 27, 22, 22, 22, 22, 22, -+ 22, 22, 22, 22, 22, 56, 56, 56, 56, 56, 56, 56, -+ 56, 56, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, -+ 68, 68, 84, 84, 84, 84, 84, 84, 91, 91, 94, 97, -+ 97, 97, 104, 104, 104, 108, 108, 108, 108, 108, 108, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, -+ 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, -+ 174, 174, 174, 174, 174, 204, 204, 204, 204, 204, 204, 174, -+ 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, 174, -+ 174, 174, 174, 174, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 300, 300, 300, 300, 300, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 400, 400, -+ 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, -+ 400, 400, 400, 400, 400, 400, 400, 400, 400, 428, 428, 428, -+ 428, 428, 428, 428, 428, 428, 428, 428, 441, 441, 441, 441, -+ 441, 441, 441, 441, 441, 428, 428, 428, 428, 428, 428, 428, -+ 428, 428, 428, 428, 428, 463, 463, 463, 463, 463, 463, 463, -+ 463, 463, 463, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, -+ 525, 525, 525, 117, 117, 117, 117, 117, 117, 117, 547, 547, -+ 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, -+ 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, -+ 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, 547, -+ 547, 547, 547, 547, 547, 547, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, -+ 117, 117, 117, 117, 117, 117, 117, 0, 0, 0, 0, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, 655, -+ 655, 655, 856, 856, 859, 862, 862, 862, 866, 866, 866, 870, -+ 870, 870, 0, 895, 898, 898, 899, 898, 898, 898, 898, 898, -+ 898, 898, 898, 912, 915, 918, 921, 922, 963, 963, 966, 969, -+ 971, 972, 975, 977, 981, 981, 984, 984, 987, 991, 993, 996, -+ 996, 1004, 1005, 1006, 1009, 1009, 1009, 0, 1016, 1016, 0, 1043, -+ 1043, 1052, 1062, 1043, 1043, 1043, 1043, 1076, 1043, 1043, 1043, 1084, -+ 1095, 1102, 1104, 1043, 1114, 1043, 1125, 1043, 1043, 1043, 1043, 1043, -+ 1146, 0, 0, 0, 1167, 1167, 1200, 1200, 1203, 1203, 1206, 1209, -+ 1211, 1213, 1216, 1216, 1224, 1225, 1225, 1225, 0, 1233, 0, 1242, -+ 1244, 1244, 1244, 0, 1254, 1256, 1256, 1256, 0, 0}; -+ -+static const int regex_start = 746; -+static const int regex_error = 0; -+ -+static const int regex_en_readVerb = 787; -+static const int regex_en_readUCP = 790; -+static const int regex_en_readBracedUCP = 559; -+static const int regex_en_readUCPSingle = 818; -+static const int regex_en_charClassGuts = 819; -+static const int regex_en_readClass = 836; -+static const int regex_en_readQuotedLiteral = 838; -+static const int regex_en_readQuotedClass = 843; -+static const int regex_en_readComment = 848; -+static const int regex_en_readNewlineTerminatedComment = 849; -+static const int regex_en_main = 746; -+ -+/** \brief Main parser call, returns root Component or nullptr. */ -+unique_ptr parse(const char *ptr, ParseMode &globalMode) { -+ assert(ptr); -+ ++const char *read_control_verbs(const char *ptr, const char *end, size_t start, ++ ParseMode &mode) { + const char *p = ptr; -+ const char *pe = ptr + strlen(ptr); -+ -+ // First, read the control verbs, set any global mode flags and move the -+ // ptr forward. -+ p = read_control_verbs(p, pe, 0, globalMode); -+ ++ const char *pe = end; + const char *eof = pe; ++ const char *ts, *te; + int cs; + UNUSED int act; -+ int top; -+ vector stack; -+ const char *ts, *te; -+ unichar accumulator = 0; -+ unichar octAccumulator = 0; /* required as we are also accumulating for -+ * back ref when looking for octals */ -+ unsigned repeatN = 0; -+ unsigned repeatM = 0; -+ string label; -+ -+ ParseMode mode = globalMode; -+ ParseMode newMode; + -+ bool negated = false; -+ bool inComment = false; -+ -+ // Stack of sequences and flags used to store state when we enter -+ // sub-sequences. -+ vector sequences; ++ static const char _ControlVerbs_actions[] = { ++ 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9}; + -+ // Index of the next capturing group. Note that zero is reserved for the -+ // root sequence. -+ unsigned groupIndex = 1; ++ static const unsigned char _ControlVerbs_key_offsets[] = { ++ 0, 7, 8, 10, 12, 14, 16, 18, 20, 21, 23, 25, 27, ++ 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 55, ++ 57, 59, 61, 63, 66, 68, 70, 72, 74, 76, 79, 82, 84, ++ 86, 88, 90, 92, 94, 96, 98, 100, 102, 105, 107, 109, 111, ++ 113, 115, 117, 119, 121, 123, 125, 127, 129, 131, 133, 135, 137, ++ 139, 141, 143, 146, 148, 149, 151, 155, 157, 159, 160, 161}; + -+ // Set storing group names that are currently in use. -+ flat_set groupNames; ++ static const char _ControlVerbs_trans_keys[] = { ++ 41, 65, 66, 67, 76, 78, 85, 41, 41, 78, 41, 89, 41, 67, 41, 82, 41, ++ 76, 41, 70, 41, 41, 83, 41, 82, 41, 95, 41, 65, 85, 41, 78, 41, 89, ++ 41, 67, 41, 78, 41, 73, 41, 67, 41, 79, 41, 68, 41, 69, 41, 82, 41, ++ 76, 41, 70, 73, 41, 77, 41, 73, 41, 84, 41, 95, 41, 77, 82, 41, 65, ++ 41, 84, 41, 67, 41, 72, 41, 61, 41, 48, 57, 41, 48, 57, 41, 69, 41, ++ 67, 41, 85, 41, 82, 41, 83, 41, 73, 41, 79, 41, 78, 41, 79, 41, 95, ++ 41, 65, 83, 41, 85, 41, 84, 41, 79, 41, 95, 41, 80, 41, 79, 41, 83, ++ 41, 83, 41, 69, 41, 83, 41, 83, 41, 84, 41, 65, 41, 82, 41, 84, 41, ++ 95, 41, 79, 41, 80, 41, 84, 41, 67, 84, 41, 80, 41, 41, 70, 41, 49, ++ 51, 56, 41, 54, 41, 50, 41, 40, 42, 0}; + -+ // Root sequence. -+ unique_ptr rootSeq = -+ ue2::make_unique(); -+ rootSeq->setCaptureIndex(0); ++ static const char _ControlVerbs_single_lengths[] = { ++ 7, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, ++ 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, ++ 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ++ 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 2, 4, 2, 2, 1, 1, 1}; + -+ // Current sequence being appended to -+ ComponentSequence *currentSeq = rootSeq.get(); ++ static const char _ControlVerbs_range_lengths[] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + -+ // The current character class being appended to. This is used as the -+ // accumulator for both character class and UCP properties. -+ unique_ptr currentCls; ++ static const short _ControlVerbs_index_offsets[] = { ++ 0, 8, 10, 13, 16, 19, 22, 25, 28, 30, 33, 36, 39, ++ 43, 46, 49, 52, 55, 58, 61, 64, 67, 70, 73, 76, 80, ++ 83, 86, 89, 92, 96, 99, 102, 105, 108, 111, 114, 117, 120, ++ 123, 126, 129, 132, 135, 138, 141, 144, 147, 151, 154, 157, 160, ++ 163, 166, 169, 172, 175, 178, 181, 184, 187, 190, 193, 196, 199, ++ 202, 205, 208, 212, 215, 217, 220, 225, 228, 231, 233, 235}; + -+ // True if the machine is currently inside a character class, i.e. square -+ // brackets [..]. -+ bool inCharClass = false; ++ static const char _ControlVerbs_indicies[] = { ++ 0, 2, 3, 4, 5, 6, 7, 1, 8, 1, 8, 9, 1, 8, 10, 1, 11, ++ 12, 1, 8, 13, 1, 8, 14, 1, 8, 15, 1, 11, 1, 8, 16, 1, 8, ++ 17, 1, 8, 18, 1, 8, 19, 20, 1, 8, 21, 1, 8, 22, 1, 8, 12, ++ 1, 8, 23, 1, 8, 24, 1, 8, 25, 1, 8, 26, 1, 8, 27, 1, 8, ++ 15, 1, 8, 28, 1, 11, 14, 1, 8, 15, 29, 1, 8, 30, 1, 8, 31, ++ 1, 8, 32, 1, 8, 33, 1, 8, 34, 35, 1, 8, 36, 1, 8, 37, 1, ++ 8, 38, 1, 8, 39, 1, 8, 40, 1, 8, 41, 1, 11, 41, 1, 8, 42, ++ 1, 8, 43, 1, 8, 44, 1, 8, 45, 1, 8, 46, 1, 8, 47, 1, 8, ++ 48, 1, 8, 39, 1, 8, 49, 1, 8, 50, 1, 8, 51, 52, 1, 8, 53, ++ 1, 8, 54, 1, 8, 55, 1, 8, 56, 1, 8, 57, 1, 8, 58, 1, 8, ++ 59, 1, 8, 60, 1, 8, 61, 1, 8, 62, 1, 8, 15, 1, 8, 63, 1, ++ 8, 64, 1, 8, 65, 1, 8, 66, 1, 8, 67, 1, 8, 68, 1, 8, 69, ++ 1, 8, 15, 1, 8, 70, 71, 1, 8, 72, 1, 73, 1, 8, 74, 1, 75, ++ 76, 77, 78, 1, 8, 15, 1, 8, 15, 1, 75, 1, 80, 79, 82, 81, 0}; + -+ // True if the machine is inside a character class but it has not processed -+ // any "real" elements yet, i.e. it's still processing meta-characters like -+ // '^'. -+ bool inCharClassEarly = false; ++ static const char _ControlVerbs_trans_targs[] = { ++ 75, 1, 2, 9, 22, 24, 45, 67, 75, 3, 4, 75, 5, 6, 7, 8, 10, ++ 11, 12, 13, 16, 14, 15, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, ++ 30, 37, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 43, 44, 46, 47, ++ 48, 59, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 63, 64, ++ 65, 66, 68, 70, 69, 75, 71, 75, 72, 73, 74, 75, 76, 75, 0}; + -+ // Location at which the current character class began. -+ const char *currentClsBegin = p; ++ static const char _ControlVerbs_trans_actions[] = { ++ 19, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 15, 5, 17, 0}; + -+ // We throw exceptions on various parsing failures beyond this point: we -+ // use a try/catch block here to clean up our allocated memory before we -+ // re-throw the exception to the caller. -+ try { -+ // Embed the Ragel machine here ++ static const char _ControlVerbs_to_state_actions[] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}; + -+ { -+ cs = regex_start; -+ top = 0; -+ ts = 0; -+ te = 0; -+ act = 0; -+ } ++ static const char _ControlVerbs_from_state_actions[] = { ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ++ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0}; ++ ++ static const short _ControlVerbs_eof_trans[] = { ++ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ++ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ++ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ++ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 82}; ++ ++ static const int ControlVerbs_start = 75; ++ static const int ControlVerbs_first_final = 75; ++ static const int ControlVerbs_error = -1; ++ ++ static const int ControlVerbs_en_main = 75; ++ ++ { ++ cs = ControlVerbs_start; ++ ts = 0; ++ te = 0; ++ act = 0; ++ } ++ ++ try { + + { + int _klen; + unsigned int _trans; -+ short _widec; -+ const short *_acts; ++ const char *_acts; + unsigned int _nacts; -+ const short *_keys; ++ const char *_keys; + + if (p == pe) + goto _test_eof; -+ if (cs == 0) -+ goto _out; + _resume: -+ _acts = _regex_actions + _regex_from_state_actions[cs]; ++ _acts = ++ _ControlVerbs_actions + _ControlVerbs_from_state_actions[cs]; + _nacts = (unsigned int)*_acts++; + while (_nacts-- > 0) { + switch (*_acts++) { -+ case 24: -+ -+ { ++ case 1: { + ts = p; + } break; + } + } + -+ _widec = (*p); -+ _klen = _regex_cond_lengths[cs]; -+ _keys = _regex_cond_keys + (_regex_cond_offsets[cs] * 2); -+ if (_klen > 0) { -+ const short *_lower = _keys; -+ const short *_mid; -+ const short *_upper = _keys + (_klen << 1) - 2; -+ while (1) { -+ if (_upper < _lower) -+ break; -+ -+ _mid = _lower + (((_upper - _lower) >> 1) & ~1); -+ if (_widec < _mid[0]) -+ _upper = _mid - 2; -+ else if (_widec > _mid[1]) -+ _lower = _mid + 2; -+ else { -+ switch (_regex_cond_spaces[_regex_cond_offsets[cs] + -+ ((_mid - _keys) >> 1)]) { -+ case 0: { -+ _widec = (short)(128 + ((*p) - -128)); -+ if ( -+ -+ mode.utf8) -+ _widec += 256; -+ break; -+ } -+ case 1: { -+ _widec = (short)(1152 + ((*p) - -128)); -+ if ( -+ -+ mode.ignore_space) -+ _widec += 256; -+ break; -+ } -+ case 2: { -+ _widec = (short)(640 + ((*p) - -128)); -+ if ( -+ -+ inCharClassEarly) -+ _widec += 256; -+ break; -+ } -+ } -+ break; -+ } -+ } -+ } -+ -+ _keys = _regex_trans_keys + _regex_key_offsets[cs]; -+ _trans = _regex_index_offsets[cs]; ++ _keys = _ControlVerbs_trans_keys + _ControlVerbs_key_offsets[cs]; ++ _trans = _ControlVerbs_index_offsets[cs]; + -+ _klen = _regex_single_lengths[cs]; ++ _klen = _ControlVerbs_single_lengths[cs]; + if (_klen > 0) { -+ const short *_lower = _keys; -+ const short *_mid; -+ const short *_upper = _keys + _klen - 1; ++ const char *_lower = _keys; ++ const char *_mid; ++ const char *_upper = _keys + _klen - 1; + while (1) { + if (_upper < _lower) + break; + + _mid = _lower + ((_upper - _lower) >> 1); -+ if (_widec < *_mid) ++ if ((*p) < *_mid) + _upper = _mid - 1; -+ else if (_widec > *_mid) ++ else if ((*p) > *_mid) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); @@ -3512,19 +1063,19 @@ index 0000000..36d0e05 + _trans += _klen; + } + -+ _klen = _regex_range_lengths[cs]; ++ _klen = _ControlVerbs_range_lengths[cs]; + if (_klen > 0) { -+ const short *_lower = _keys; -+ const short *_mid; -+ const short *_upper = _keys + (_klen << 1) - 2; ++ const char *_lower = _keys; ++ const char *_mid; ++ const char *_upper = _keys + (_klen << 1) - 2; + while (1) { + if (_upper < _lower) + break; + + _mid = _lower + (((_upper - _lower) >> 1) & ~1); -+ if (_widec < _mid[0]) ++ if ((*p) < _mid[0]) + _upper = _mid - 2; -+ else if (_widec > _mid[1]) ++ else if ((*p) > _mid[1]) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys) >> 1); @@ -3535,7929 +1086,347 @@ index 0000000..36d0e05 + } + + _match: -+ _trans = _regex_indicies[_trans]; ++ _trans = _ControlVerbs_indicies[_trans]; + _eof_trans: -+ cs = _regex_trans_targs[_trans]; ++ cs = _ControlVerbs_trans_targs[_trans]; + -+ if (_regex_trans_actions[_trans] == 0) ++ if (_ControlVerbs_trans_actions[_trans] == 0) + goto _again; + -+ _acts = _regex_actions + _regex_trans_actions[_trans]; ++ _acts = _ControlVerbs_actions + _ControlVerbs_trans_actions[_trans]; + _nacts = (unsigned int)*_acts++; + while (_nacts-- > 0) { + switch (*_acts++) { -+ case 0: -+ -+ { -+ label.clear(); -+ } break; -+ case 1: -+ -+ { -+ label.push_back((*p)); -+ } break; -+ case 2: -+ -+ { -+ octAccumulator = 0; -+ } break; -+ case 3: -+ -+ { -+ accumulator = 0; -+ } break; -+ case 4: -+ -+ { -+ octAccumulator = 0; -+ pushOct(&octAccumulator, (*p)); -+ } break; -+ case 5: -+ -+ { -+ accumulator = 0; -+ pushDec(&accumulator, (*p)); -+ } break; -+ case 6: -+ -+ { -+ repeatN = 0; -+ repeatM = 0; -+ } break; -+ case 7: -+ -+ { -+ pushDec(&repeatN, (*p)); -+ } break; -+ case 8: -+ -+ { -+ pushDec(&repeatM, (*p)); -+ } break; -+ case 9: -+ -+ { -+ pushOct(&octAccumulator, (*p)); -+ } break; -+ case 10: -+ -+ { -+ pushDec(&accumulator, (*p)); -+ } break; -+ case 11: -+ -+ { -+ accumulator *= 16; -+ accumulator += (*p) - '0'; -+ } break; -+ case 12: -+ -+ { -+ accumulator *= 16; -+ accumulator += 10 + (*p) - 'a'; ++ case 2: { ++ te = p + 1; + } break; -+ case 13: -+ -+ { -+ accumulator *= 16; -+ accumulator += 10 + (*p) - 'A'; ++ case 3: { ++ te = p + 1; ++ { mode.utf8 = true; } + } break; -+ case 14: -+ -+ { -+ newMode = mode; ++ case 4: { ++ te = p + 1; ++ { mode.ucp = true; } + } break; -+ case 15: -+ -+ { -+ switch ((*p)) { -+ case 'i': -+ newMode.caseless = true; -+ break; -+ case 'm': -+ newMode.multiline = true; -+ break; -+ case 's': -+ newMode.dotall = true; -+ break; -+ case 'x': -+ newMode.ignore_space = true; -+ break; -+ default: -+ assert(0); // this action only called for [imsx] -+ break; ++ case 5: { ++ te = p + 1; ++ { ++ ostringstream str; ++ str << "Unsupported control verb " ++ << string(ts, te - ts); ++ throw LocatedParseError(str.str()); + } + } break; -+ case 16: -+ -+ { -+ switch ((*p)) { -+ case 'i': -+ newMode.caseless = false; -+ break; -+ case 'm': -+ newMode.multiline = false; -+ break; -+ case 's': -+ newMode.dotall = false; -+ break; -+ case 'x': -+ newMode.ignore_space = false; -+ break; -+ default: -+ assert(0); // this action only called for [imsx] -+ break; ++ case 6: { ++ te = p + 1; ++ { ++ ostringstream str; ++ str << "Unknown control verb " << string(ts, te - ts); ++ throw LocatedParseError(str.str()); + } + } break; -+ case 17: -+ -+ { -+ repeatM = repeatN; -+ } break; -+ case 18: -+ -+ { -+ repeatM = ComponentRepeat::NoLimit; -+ } break; -+ case 19: -+ -+ { -+ negated = !negated; ++ case 7: { ++ te = p + 1; ++ { ++ p--; ++ { ++ p++; ++ goto _out; ++ } ++ } + } break; -+ case 20: -+ -+ { ++ case 8: { ++ te = p; + p--; + { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } ++ p--; + { -+ stack[top++] = cs; -+ cs = 790; -+ goto _again; ++ p++; ++ goto _out; + } + } + } break; -+ case 21: -+ -+ { -+ if (!inCharClass) { // not inside [..] -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } ++ case 9: { ++ { p = ((te)) - 1; } + { -+ cs = stack[--top]; -+ goto _again; ++ p--; ++ { ++ p++; ++ goto _out; ++ } + } + } break; -+ case 22: -+ -+ { -+ throw LocatedParseError("Malformed property"); -+ } break; -+ case 25: -+ -+ { -+ te = p + 1; -+ } break; -+ case 26: ++ } ++ } + -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("(*UTF8) must be at start of " -+ "expression, encountered"); -+ } -+ } break; -+ case 27: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("(*UTF) must be at start of " -+ "expression, encountered"); -+ } -+ } break; -+ case 28: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("(*UCP) must be at start of " -+ "expression, encountered"); -+ } -+ } break; -+ case 29: -+ -+ { -+ te = p + 1; -+ { -+ ParseMode temp_mode; -+ assert(ts - 2 >= -+ ptr); // parser needs the '(*' at the start too. -+ read_control_verbs(ts - 2, te, (ts - 2 - ptr), -+ temp_mode); -+ assert(0); // Should have thrown a parse error. -+ throw LocatedParseError("Unknown control verb"); -+ } -+ } break; -+ case 30: -+ -+ { -+ te = p + 1; -+ { throw LocatedParseError("Unknown control verb"); } -+ } break; -+ case 31: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Unknown control verb"); } -+ } break; -+ case 32: -+ -+ { -+ { p = ((te)) - 1; } -+ { throw LocatedParseError("Unknown control verb"); } -+ } break; -+ case 33: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_CC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 34: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_CF, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 35: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_CN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 36: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_CS, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 37: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_LL, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 38: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_LM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 39: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_LO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 40: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_LT, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 41: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_LU, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 42: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_L_AND, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 43: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_MC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 44: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_MN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 45: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_ND, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } ++ _again: ++ _acts = _ControlVerbs_actions + _ControlVerbs_to_state_actions[cs]; ++ _nacts = (unsigned int)*_acts++; ++ while (_nacts-- > 0) { ++ switch (*_acts++) { ++ case 0: { ++ ts = 0; + } break; -+ case 46: ++ } ++ } + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_NL, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 47: ++ if (++p != pe) ++ goto _resume; ++ _test_eof : {} ++ if (p == eof) { ++ if (_ControlVerbs_eof_trans[cs] > 0) { ++ _trans = _ControlVerbs_eof_trans[cs] - 1; ++ goto _eof_trans; ++ } ++ } + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_NO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 48: ++ _out : {} ++ } + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 49: ++ } catch (LocatedParseError &error) { ++ if (ts >= ptr && ts <= pe) { ++ error.locate(ts - ptr + start); ++ } else { ++ error.locate(0); ++ } ++ throw; ++ } + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PD, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 50: ++ return p; ++} + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 51: ++} // namespace ue2 +diff --git a/src/rose/counting_miracle.h b/src/rose/counting_miracle.h +index 976208b..4456679 100644 +--- a/src/rose/counting_miracle.h ++++ b/src/rose/counting_miracle.h +@@ -94,7 +94,7 @@ u32 roseCountingMiracleScanShufti(m128 mask_lo, m128 mask_hi, u8 poison, + u32 count = *count_inout; + + const m128 zeroes = zeroes128(); +- const m128 low4bits = _mm_set1_epi8(0xf); ++ const m128 low4bits = set16x8(0xf); + + for (; d + 16 <= d_end; d_end -= 16) { + m128 data = loadu128(d_end - 16); +diff --git a/src/util/arch.h b/src/util/arch.h +index 985fec6..fe4a910 100644 +--- a/src/util/arch.h ++++ b/src/util/arch.h +@@ -61,6 +61,10 @@ + #define HAVE_AVX512VBMI + #endif + ++#if defined(__aarch64__) ++#define HAVE_NEON ++#endif + -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PF, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 52: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 53: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 54: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_PS, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 55: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_SC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 56: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_SK, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 57: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_SM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 58: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_SO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 59: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_ZL, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 60: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_ZP, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 61: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_ZS, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 62: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_XAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 63: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_XPS, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 64: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_XSP, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 65: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_XWD, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 66: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_ARABIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 67: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 68: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_AVESTAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 69: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BALINESE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 70: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BAMUM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 71: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BATAK, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 72: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BENGALI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 73: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 74: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BRAHMI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 75: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BRAILLE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 76: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BUGINESE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 77: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_BUHID, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 78: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, -+ negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 79: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CARIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 80: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CHAM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 81: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 82: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_COMMON, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 83: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_COPTIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 84: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 85: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 86: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 87: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_DESERET, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 88: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 89: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, -+ negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 90: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 91: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 92: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 93: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GOTHIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 94: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GREEK, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 95: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GUJARATI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 96: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 97: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_HANGUL, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 98: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_HANUNOO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 99: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_HEBREW, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 100: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 101: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 102: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_INHERITED, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 103: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, -+ negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 104: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, -+ negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 105: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_JAVANESE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 106: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KAITHI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 107: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KANNADA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 108: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KATAKANA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 109: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 110: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 111: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_KHMER, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 112: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LAO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 113: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LATIN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 114: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LEPCHA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 115: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LIMBU, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 116: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 117: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LISU, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 118: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LYCIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 119: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_LYDIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 120: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 121: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_MANDAIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 122: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 123: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 124: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_MYANMAR, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 125: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 126: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_NKO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 127: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OGHAM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 128: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 129: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 130: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 131: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, -+ negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 132: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 133: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_ORIYA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 134: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_OSMANYA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 135: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 136: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 137: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_REJANG, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 138: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_RUNIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 139: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 140: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 141: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 142: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SINHALA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 143: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 144: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 145: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_SYRIAC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 146: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAGALOG, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 147: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 148: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAI_LE, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 149: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 150: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 151: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TAMIL, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 152: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TELUGU, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 153: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_THAANA, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 154: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_THAI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 155: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TIBETAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 156: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 157: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_UGARITIC, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 158: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_VAI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 159: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_SCRIPT_YI, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 160: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_ANY, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 161: -+ -+ { -+ te = p + 1; -+ { throw LocatedParseError("Unknown property"); } -+ } break; -+ case 162: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_C, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 163: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_CO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 164: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_L, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 165: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_M, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 166: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_ME, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 167: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_N, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 168: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_P, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 169: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_S, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 170: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_UCP_Z, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 171: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(CLASS_SCRIPT_HAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 172: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Unknown property"); } -+ } break; -+ case 173: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_C, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 174: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_CO, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 175: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_L, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 176: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_M, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 177: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_ME, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 178: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_N, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 179: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_P, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 180: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_UCP_S, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 181: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ currentCls->add(CLASS_SCRIPT_HAN, negated); -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 182: -+ -+ { -+ { p = ((te)) - 1; } -+ { throw LocatedParseError("Unknown property"); } -+ } break; -+ case 183: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_C, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 184: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_L, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 185: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_M, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 186: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_N, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 187: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_P, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 188: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_S, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 189: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(CLASS_UCP_Z, negated); -+ if (!inCharClass) { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ } -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 190: -+ -+ { -+ te = p + 1; -+ { throw LocatedParseError("Unknown property"); } -+ } break; -+ case 191: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("Unsupported POSIX collating " -+ "element"); -+ } -+ } break; -+ case 192: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ALNUM, false); } -+ } break; -+ case 193: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ALNUM, true); } -+ } break; -+ case 194: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ALPHA, false); } -+ } break; -+ case 195: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ALPHA, true); } -+ } break; -+ case 196: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ASCII, false); } -+ } break; -+ case 197: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_ASCII, true); } -+ } break; -+ case 198: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_BLANK, false); } -+ } break; -+ case 199: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_BLANK, true); } -+ } break; -+ case 200: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_CNTRL, false); } -+ } break; -+ case 201: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_CNTRL, true); } -+ } break; -+ case 202: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_DIGIT, false); } -+ } break; -+ case 203: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_DIGIT, true); } -+ } break; -+ case 204: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_GRAPH, false); } -+ } break; -+ case 205: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_GRAPH, true); } -+ } break; -+ case 206: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_LOWER, false); } -+ } break; -+ case 207: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_LOWER, true); } -+ } break; -+ case 208: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_PRINT, false); } -+ } break; -+ case 209: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_PRINT, true); } -+ } break; -+ case 210: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_PUNCT, false); } -+ } break; -+ case 211: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_PUNCT, true); } -+ } break; -+ case 212: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_SPACE, false); } -+ } break; -+ case 213: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_SPACE, true); } -+ } break; -+ case 214: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_UPPER, false); } -+ } break; -+ case 215: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_UPPER, true); } -+ } break; -+ case 216: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_WORD, false); } -+ } break; -+ case 217: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_WORD, true); } -+ } break; -+ case 218: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_XDIGIT, false); } -+ } break; -+ case 219: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_XDIGIT, true); } -+ } break; -+ case 220: -+ -+ { -+ te = p + 1; -+ { throw LocatedParseError("Invalid POSIX named class"); } -+ } break; -+ case 221: -+ -+ { -+ te = p + 1; -+ { -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 843; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 222: -+ -+ { -+ te = p + 1; -+ { /*noop*/ -+ } -+ } break; -+ case 223: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x08'); } -+ } break; -+ case 224: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x09'); } -+ } break; -+ case 225: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x0a'); } -+ } break; -+ case 226: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x0d'); } -+ } break; -+ case 227: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x0c'); } -+ } break; -+ case 228: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x07'); } -+ } break; -+ case 229: -+ -+ { -+ te = p + 1; -+ { currentCls->add('\x1b'); } -+ } break; -+ case 230: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_HORZ, false); } -+ } break; -+ case 231: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_HORZ, true); } -+ } break; -+ case 232: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_VERT, false); } -+ } break; -+ case 233: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_VERT, true); } -+ } break; -+ case 234: -+ -+ { -+ te = p + 1; -+ { -+ negated = false; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 559; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 235: -+ -+ { -+ te = p + 1; -+ { -+ negated = false; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 818; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 236: -+ -+ { -+ te = p + 1; -+ { -+ negated = true; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 559; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 237: -+ -+ { -+ te = p + 1; -+ { -+ negated = true; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 818; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 238: -+ -+ { -+ te = p + 1; -+ { currentCls->add(octAccumulator); } -+ } break; -+ case 239: -+ -+ { -+ te = p + 1; -+ { currentCls->add(octAccumulator); } -+ } break; -+ case 240: -+ -+ { -+ te = p + 1; -+ { -+ string oct(ts + 3, te - ts - 4); -+ unsigned long val; -+ try { -+ val = stoul(oct, nullptr, 8); -+ } catch (const std::out_of_range &) { -+ val = MAX_UNICODE + 1; -+ } -+ if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { -+ throw LocatedParseError( -+ "Value in \\o{...} sequence is too large"); -+ } -+ currentCls->add((unichar)val); -+ } -+ } break; -+ case 241: -+ -+ { -+ te = p + 1; -+ { currentCls->add(accumulator); } -+ } break; -+ case 242: -+ -+ { -+ te = p + 1; -+ { -+ // whatever we found here -+ currentCls->add(*(ts + 1)); -+ } -+ } break; -+ case 243: -+ -+ { -+ te = p + 1; -+ { -+ string hex(ts + 3, te - ts - 4); -+ unsigned long val; -+ try { -+ val = stoul(hex, nullptr, 16); -+ } catch (const std::out_of_range &) { -+ val = MAX_UNICODE + 1; -+ } -+ if (val > MAX_UNICODE) { -+ throw LocatedParseError( -+ "Value in \\x{...} sequence is too large"); -+ } -+ currentCls->add((unichar)val); -+ } -+ } break; -+ case 244: -+ -+ { -+ te = p + 1; -+ { -+ if (te - ts < 3) { -+ assert(te - ts == 2); -+ throw LocatedParseError(SLASH_C_ERROR); -+ } else { -+ assert(te - ts == 3); -+ currentCls->add(decodeCtrl(ts[2])); -+ } -+ } -+ } break; -+ case 245: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_WORD, false); } -+ } break; -+ case 246: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_WORD, true); } -+ } break; -+ case 247: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_SPACE, false); } -+ } break; -+ case 248: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_SPACE, true); } -+ } break; -+ case 249: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_DIGIT, false); } -+ } break; -+ case 250: -+ -+ { -+ te = p + 1; -+ { currentCls->add(CLASS_DIGIT, true); } -+ } break; -+ case 251: -+ -+ { -+ te = p + 1; -+ { currentCls->addDash(); } -+ } break; -+ case 252: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "'\\" << *(ts + 1) << "' at index " << ts - ptr -+ << " not supported in a character class."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 253: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "'\\" << *(ts + 1) << "' at index " << ts - ptr -+ << " not supported in a character class."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 254: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "'\\" << *(ts + 1) << "' at index " << ts - ptr -+ << " not supported in a character class."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 255: -+ -+ { -+ te = p + 1; -+ { -+ // add the literal char -+ currentCls->add(*(ts + 1)); -+ } -+ } break; -+ case 256: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint2c(ts)); -+ } -+ } break; -+ case 257: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint3c(ts)); -+ } -+ } break; -+ case 258: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint4c(ts)); -+ } -+ } break; -+ case 259: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 260: -+ -+ { -+ te = p + 1; -+ { currentCls->add((u8)*ts); } -+ } break; -+ case 261: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->finalize(); -+ currentSeq->addComponent(move(currentCls)); -+ inCharClass = false; -+ { -+ cs = 746; -+ goto _again; -+ } -+ } -+ } break; -+ case 262: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Malformed property"); } -+ } break; -+ case 263: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Malformed property"); } -+ } break; -+ case 264: -+ -+ { -+ te = p; -+ p--; -+ { currentCls->add(octAccumulator); } -+ } break; -+ case 265: -+ -+ { -+ te = p; -+ p--; -+ { currentCls->add(octAccumulator); } -+ } break; -+ case 266: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError("Value in \\o{...} sequence is " -+ "non-octal or missing braces"); -+ } -+ } break; -+ case 267: -+ -+ { -+ te = p; -+ p--; -+ { currentCls->add(accumulator); } -+ } break; -+ case 268: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError("Value in \\x{...} sequence is " -+ "non-hex or missing }"); -+ } -+ } break; -+ case 269: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (te - ts < 3) { -+ assert(te - ts == 2); -+ throw LocatedParseError(SLASH_C_ERROR); -+ } else { -+ assert(te - ts == 3); -+ currentCls->add(decodeCtrl(ts[2])); -+ } -+ } -+ } break; -+ case 270: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 271: -+ -+ { -+ te = p; -+ p--; -+ { currentCls->add((u8)*ts); } -+ } break; -+ case 272: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError("Value in \\o{...} sequence is " -+ "non-octal or missing braces"); -+ } -+ } break; -+ case 273: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError("Value in \\x{...} sequence is " -+ "non-hex or missing }"); -+ } -+ } break; -+ case 274: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 275: -+ -+ { -+ { p = ((te)) - 1; } -+ { currentCls->add((u8)*ts); } -+ } break; -+ case 276: -+ -+ { -+ te = p + 1; -+ { -+ if (currentCls->isNegated()) { -+ // Already seen a caret; the second one is not a -+ // meta-character. -+ inCharClassEarly = false; -+ p--; -+ { -+ cs = 819; -+ goto _again; -+ } -+ } else { -+ currentCls->negate(); -+ // Note: we cannot switch off inCharClassEarly here, -+ // as /[^]]/ needs to use the right square bracket -+ // path below. -+ } -+ } -+ } break; -+ case 277: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(']'); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 278: -+ -+ { -+ te = p + 1; -+ { -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 843; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 279: -+ -+ { -+ te = p + 1; -+ { /*noop*/ -+ } -+ } break; -+ case 280: -+ -+ { -+ te = p + 1; -+ { -+ inCharClassEarly = false; -+ p--; -+ { -+ cs = 819; -+ goto _again; -+ } -+ } -+ } break; -+ case 281: -+ -+ { -+ te = p; -+ p--; -+ { -+ inCharClassEarly = false; -+ p--; -+ { -+ cs = 819; -+ goto _again; -+ } -+ } -+ } break; -+ case 282: -+ -+ { -+ te = p + 1; -+ { -+ { -+ cs = 746; -+ goto _again; -+ } -+ } -+ } break; -+ case 283: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint2c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 284: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint3c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 285: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint4c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 286: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 287: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, *ts, mode); } -+ } break; -+ case 288: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 289: -+ -+ { -+ te = p; -+ p--; -+ { addLiteral(currentSeq, *ts, mode); } -+ } break; -+ case 290: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 291: -+ -+ { -+ te = p + 1; -+ { -+ { -+ cs = stack[--top]; -+ goto _again; -+ } -+ } -+ } break; -+ case 292: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint2c(ts)); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 293: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint3c(ts)); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 294: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ currentCls->add(readUtf8CodePoint4c(ts)); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 295: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 296: -+ -+ { -+ te = p + 1; -+ { -+ currentCls->add(*ts); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 297: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 298: -+ -+ { -+ te = p; -+ p--; -+ { -+ currentCls->add(*ts); -+ inCharClassEarly = false; -+ } -+ } break; -+ case 299: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 300: -+ -+ { -+ te = p + 1; -+ { -+ inComment = false; -+ { -+ cs = 746; -+ goto _again; -+ } -+ } -+ } break; -+ case 301: -+ -+ { -+ te = p + 1; -+ } break; -+ case 302: -+ -+ { -+ te = p + 1; -+ { -+ inComment = false; -+ { -+ cs = 746; -+ goto _again; -+ } -+ } -+ } break; -+ case 303: -+ -+ { -+ te = p + 1; -+ } break; -+ case 304: -+ -+ { -+ act = 288; -+ } break; -+ case 305: -+ -+ { -+ act = 290; -+ } break; -+ case 306: -+ -+ { -+ act = 330; -+ } break; -+ case 307: -+ -+ { -+ te = p + 1; -+ { -+ if (sequences.empty()) { -+ throw LocatedParseError("Unmatched parentheses"); -+ } -+ currentSeq->finalize(); -+ POP_SEQUENCE; -+ } -+ } break; -+ case 308: -+ -+ { -+ te = p + 1; -+ { currentSeq->addAlternation(); } -+ } break; -+ case 309: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("POSIX named classes are only " -+ "supported inside a class"); -+ } -+ } break; -+ case 310: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError("Unsupported POSIX collating " -+ "element"); -+ } -+ } break; -+ case 311: -+ -+ { -+ te = p + 1; -+ { -+ { -+ cs = 838; -+ goto _again; -+ } -+ } -+ } break; -+ case 312: -+ -+ { -+ te = p + 1; -+ { /* noop */ -+ } -+ } break; -+ case 313: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ generateComponent(CLASS_ANY, false, mode)); -+ } -+ } break; -+ case 314: -+ -+ { -+ te = p + 1; -+ { -+ if (mode.utf8) { -+ throw LocatedParseError( -+ "\\C is unsupported in UTF8"); -+ } -+ currentSeq->addComponent( -+ ue2::make_unique()); -+ } -+ } break; -+ case 315: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 0, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_NONGREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 316: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 0, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_POSSESSIVE)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 317: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 1, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_NONGREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 318: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 1, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_POSSESSIVE)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 319: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 320: -+ -+ { -+ te = p + 1; -+ { -+ if (!currentSeq->addRepeat( -+ 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 321: -+ -+ { -+ te = p + 1; -+ { -+ if (repeatN > repeatM || repeatM == 0) { -+ throwInvalidRepeat(); -+ } else if (!currentSeq->addRepeat( -+ repeatN, repeatM, -+ ComponentRepeat::REPEAT_NONGREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 322: -+ -+ { -+ te = p + 1; -+ { -+ if (repeatN > repeatM || repeatM == 0) { -+ throwInvalidRepeat(); -+ } else if (!currentSeq->addRepeat( -+ repeatN, repeatM, -+ ComponentRepeat::REPEAT_POSSESSIVE)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 323: -+ -+ { -+ te = p + 1; -+ { -+ inComment = true; -+ { -+ cs = 849; -+ goto _again; -+ } -+ } -+ } break; -+ case 324: -+ -+ { -+ te = p + 1; -+ { -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 787; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 325: -+ -+ { -+ te = p + 1; -+ { -+ assert(0); -+ { -+ p++; -+ goto _out; -+ } -+ } -+ } break; -+ case 326: -+ -+ { -+ te = p + 1; -+ { -+ auto bound = mode.multiline -+ ? ComponentBoundary::BEGIN_LINE -+ : ComponentBoundary::BEGIN_STRING; -+ currentSeq->addComponent( -+ ue2::make_unique(bound)); -+ } -+ } break; -+ case 327: -+ -+ { -+ te = p + 1; -+ { -+ auto bound = -+ mode.multiline -+ ? ComponentBoundary::END_LINE -+ : ComponentBoundary::END_STRING_OPTIONAL_LF; -+ currentSeq->addComponent( -+ ue2::make_unique(bound)); -+ } -+ } break; -+ case 328: -+ -+ { -+ te = p + 1; -+ { -+ auto bound = ComponentBoundary::BEGIN_STRING; -+ currentSeq->addComponent( -+ ue2::make_unique(bound)); -+ } -+ } break; -+ case 329: -+ -+ { -+ te = p + 1; -+ { -+ auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; -+ currentSeq->addComponent( -+ ue2::make_unique(bound)); -+ } -+ } break; -+ case 330: -+ -+ { -+ te = p + 1; -+ { -+ auto bound = ComponentBoundary::END_STRING; -+ currentSeq->addComponent( -+ ue2::make_unique(bound)); -+ } -+ } break; -+ case 331: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique( -+ ts - ptr, false, mode)); -+ } -+ } break; -+ case 332: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique( -+ ts - ptr, true, mode)); -+ } -+ } break; -+ case 333: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x09', mode); } -+ } break; -+ case 334: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x0a', mode); } -+ } break; -+ case 335: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x0d', mode); } -+ } break; -+ case 336: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x0c', mode); } -+ } break; -+ case 337: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x07', mode); } -+ } break; -+ case 338: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, '\x1b', mode); } -+ } break; -+ case 339: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, octAccumulator, mode); } -+ } break; -+ case 340: -+ -+ { -+ te = p + 1; -+ { -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } -+ } break; -+ case 341: -+ -+ { -+ te = p + 1; -+ { -+ // Accumulator is a negative offset. -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ if (accumulator >= groupIndex) { -+ throw LocatedParseError("Invalid reference"); -+ } -+ unsigned idx = groupIndex - accumulator; -+ currentSeq->addComponent( -+ ue2::make_unique(idx)); -+ } -+ } break; -+ case 342: -+ -+ { -+ te = p + 1; -+ { -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } -+ } break; -+ case 343: -+ -+ { -+ te = p + 1; -+ { -+ // Accumulator is a negative offset. -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ if (accumulator >= groupIndex) { -+ throw LocatedParseError("Invalid reference"); -+ } -+ unsigned idx = groupIndex - accumulator; -+ currentSeq->addComponent( -+ ue2::make_unique(idx)); -+ } -+ } break; -+ case 344: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 345: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 346: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 347: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 348: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 349: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "Onigiruma subroutine call at index " << ts - ptr -+ << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 350: -+ -+ { -+ te = p + 1; -+ { -+ string oct(ts + 3, te - ts - 4); -+ unsigned long val; -+ try { -+ val = stoul(oct, nullptr, 8); -+ } catch (const std::out_of_range &) { -+ val = MAX_UNICODE + 1; -+ } -+ if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { -+ throw LocatedParseError( -+ "Value in \\o{...} sequence is too large"); -+ } -+ addEscapedOctal(currentSeq, (unichar)val, mode); -+ } -+ } break; -+ case 351: -+ -+ { -+ te = p + 1; -+ { addEscapedHex(currentSeq, accumulator, mode); } -+ } break; -+ case 352: -+ -+ { -+ te = p + 1; -+ { -+ string hex(ts + 3, te - ts - 4); -+ unsigned long val; -+ try { -+ val = stoul(hex, nullptr, 16); -+ } catch (const std::out_of_range &) { -+ val = MAX_UNICODE + 1; -+ } -+ if (val > MAX_UNICODE) { -+ throw LocatedParseError( -+ "Value in \\x{...} sequence is too large"); -+ } -+ addEscapedHex(currentSeq, (unichar)val, mode); -+ } -+ } break; -+ case 353: -+ -+ { -+ te = p + 1; -+ { -+ if (te - ts < 3) { -+ assert(te - ts == 2); -+ throw LocatedParseError(SLASH_C_ERROR); -+ } else { -+ assert(te - ts == 3); -+ addLiteral(currentSeq, decodeCtrl(ts[2]), mode); -+ } -+ } -+ } break; -+ case 354: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "'\\" << *(ts + 1) << "' at index " << ts - ptr -+ << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 355: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_WORD, false, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 356: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_WORD, true, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 357: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_SPACE, false, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 358: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_SPACE, true, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 359: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_DIGIT, false, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 360: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_DIGIT, true, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 361: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_HORZ, false, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 362: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_HORZ, true, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 363: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_VERT, false, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 364: -+ -+ { -+ te = p + 1; -+ { -+ auto cc = generateComponent(CLASS_VERT, true, mode); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 365: -+ -+ { -+ te = p + 1; -+ { -+ assert(!currentCls && !inCharClass); -+ currentCls = getComponentClass(mode); -+ negated = false; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 559; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 366: -+ -+ { -+ te = p + 1; -+ { -+ assert(!currentCls && !inCharClass); -+ currentCls = getComponentClass(mode); -+ negated = false; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 818; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 367: -+ -+ { -+ te = p + 1; -+ { -+ assert(!currentCls && !inCharClass); -+ currentCls = getComponentClass(mode); -+ negated = true; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 559; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 368: -+ -+ { -+ te = p + 1; -+ { -+ assert(!currentCls && !inCharClass); -+ currentCls = getComponentClass(mode); -+ negated = true; -+ p--; -+ { -+ DEBUG_PRINTF("stack %zu top %d\n", stack.size(), -+ top); -+ if ((int)stack.size() == top) { -+ stack.resize(2 * (top + 1)); -+ } -+ { -+ stack[top++] = cs; -+ cs = 818; -+ goto _again; -+ } -+ } -+ } -+ } break; -+ case 369: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "\\R at index " << ts - ptr << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 370: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "\\K at index " << ts - ptr << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 371: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "\\G at index " << ts - ptr << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 372: -+ -+ { -+ te = p + 1; -+ { -+ currentSeq->addComponent( -+ ue2::make_unique(ts - ptr, mode)); -+ } -+ } break; -+ case 373: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, *(ts + 1), mode); } -+ } break; -+ case 374: -+ -+ { -+ te = p + 1; -+ { -+ inComment = true; -+ { -+ cs = 848; -+ goto _again; -+ } -+ } -+ } break; -+ case 375: -+ -+ { -+ te = p + 1; -+ { -+ mode = newMode; -+ currentSeq->addComponent( -+ ue2::make_unique()); -+ } -+ } break; -+ case 376: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ mode = newMode; -+ currentSeq = enterSequence( -+ currentSeq, ue2::make_unique()); -+ } -+ } break; -+ case 377: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, ue2::make_unique( -+ ComponentAssertion::LOOKAHEAD, -+ ComponentAssertion::POS)); -+ } -+ } break; -+ case 378: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, ue2::make_unique( -+ ComponentAssertion::LOOKAHEAD, -+ ComponentAssertion::NEG)); -+ } -+ } break; -+ case 379: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, ue2::make_unique( -+ ComponentAssertion::LOOKBEHIND, -+ ComponentAssertion::POS)); -+ } -+ } break; -+ case 380: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, ue2::make_unique( -+ ComponentAssertion::LOOKBEHIND, -+ ComponentAssertion::NEG)); -+ } -+ } break; -+ case 381: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Embedded code is not supported"); -+ } -+ } break; -+ case 382: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Embedded code is not supported"); -+ } -+ } break; -+ case 383: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique()); -+ } -+ } break; -+ case 384: -+ -+ { -+ te = p + 1; -+ { -+ assert( -+ !label.empty()); // should be guaranteed by machine -+ char c = *label.begin(); -+ if (c >= '0' && c <= '9') { -+ throw LocatedParseError( -+ "Group name cannot begin with a digit"); -+ } -+ if (!groupNames.insert(label).second) { -+ throw LocatedParseError( -+ "Two named subpatterns use the name '" + label + -+ "'"); -+ } -+ PUSH_SEQUENCE; -+ auto seq = ue2::make_unique(); -+ seq->setCaptureIndex(groupIndex++); -+ seq->setCaptureName(label); -+ currentSeq = enterSequence(currentSeq, move(seq)); -+ } -+ } break; -+ case 385: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Subpattern reference unsupported"); -+ } -+ } break; -+ case 386: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Subpattern reference unsupported"); -+ } -+ } break; -+ case 387: -+ -+ { -+ te = p + 1; -+ { -+ auto a = ue2::make_unique( -+ ComponentAssertion::LOOKAHEAD, -+ ComponentAssertion::POS); -+ ComponentAssertion *a_seq = a.get(); -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique(move(a))); -+ PUSH_SEQUENCE; -+ currentSeq = a_seq; -+ } -+ } break; -+ case 388: -+ -+ { -+ te = p + 1; -+ { -+ auto a = ue2::make_unique( -+ ComponentAssertion::LOOKAHEAD, -+ ComponentAssertion::NEG); -+ ComponentAssertion *a_seq = a.get(); -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique(move(a))); -+ PUSH_SEQUENCE; -+ currentSeq = a_seq; -+ } -+ } break; -+ case 389: -+ -+ { -+ te = p + 1; -+ { -+ auto a = ue2::make_unique( -+ ComponentAssertion::LOOKBEHIND, -+ ComponentAssertion::POS); -+ ComponentAssertion *a_seq = a.get(); -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique(move(a))); -+ PUSH_SEQUENCE; -+ currentSeq = a_seq; -+ } -+ } break; -+ case 390: -+ -+ { -+ te = p + 1; -+ { -+ auto a = ue2::make_unique( -+ ComponentAssertion::LOOKBEHIND, -+ ComponentAssertion::NEG); -+ ComponentAssertion *a_seq = a.get(); -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique(move(a))); -+ PUSH_SEQUENCE; -+ currentSeq = a_seq; -+ } -+ } break; -+ case 391: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Pattern recursion not supported"); -+ } -+ } break; -+ case 392: -+ -+ { -+ te = p + 1; -+ { -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ PUSH_SEQUENCE; -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique( -+ accumulator)); -+ } -+ } break; -+ case 393: -+ -+ { -+ te = p + 1; -+ { -+ PUSH_SEQUENCE; -+ assert(!label.empty()); -+ currentSeq = enterSequence( -+ currentSeq, -+ ue2::make_unique(label)); -+ } -+ } break; -+ case 394: -+ -+ { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "Callout at index " << ts - ptr -+ << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 395: -+ -+ { -+ te = p + 1; -+ { -+ throw LocatedParseError( -+ "Unrecognised character after (?"); -+ } -+ } break; -+ case 396: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint2c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 397: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint3c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 398: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ /* leverage ComponentClass to generate the vertices */ -+ auto cc = getComponentClass(mode); -+ cc->add(readUtf8CodePoint4c(ts)); -+ cc->finalize(); -+ currentSeq->addComponent(move(cc)); -+ } -+ } break; -+ case 399: -+ -+ { -+ te = p + 1; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 400: -+ -+ { -+ te = p + 1; -+ { -+ if (mode.ignore_space == false) { -+ addLiteral(currentSeq, *ts, mode); -+ } -+ } -+ } break; -+ case 401: -+ -+ { -+ te = p + 1; -+ { addLiteral(currentSeq, *ts, mode); } -+ } break; -+ case 402: -+ -+ { -+ te = p; -+ p--; -+ { -+ PUSH_SEQUENCE; -+ auto seq = ue2::make_unique(); -+ seq->setCaptureIndex(groupIndex++); -+ currentSeq = enterSequence(currentSeq, move(seq)); -+ } -+ } break; -+ case 403: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(!currentCls); -+ assert(!inCharClass); // not reentrant -+ currentCls = getComponentClass(mode); -+ inCharClass = true; -+ inCharClassEarly = true; -+ currentClsBegin = ts; -+ { -+ cs = 836; -+ goto _again; -+ } -+ } -+ } break; -+ case 404: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (!currentSeq->addRepeat( -+ 0, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_GREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 405: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (!currentSeq->addRepeat( -+ 1, ComponentRepeat::NoLimit, -+ ComponentRepeat::REPEAT_GREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 406: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (!currentSeq->addRepeat( -+ 0, 1, ComponentRepeat::REPEAT_GREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 407: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (repeatN > repeatM || repeatM == 0) { -+ throwInvalidRepeat(); -+ } else if (!currentSeq->addRepeat( -+ repeatN, repeatM, -+ ComponentRepeat::REPEAT_GREEDY)) { -+ throwInvalidRepeat(); -+ } -+ } -+ } break; -+ case 408: -+ -+ { -+ te = p; -+ p--; -+ { addLiteral(currentSeq, octAccumulator, mode); } -+ } break; -+ case 409: -+ -+ { -+ te = p; -+ p--; -+ { -+ // If there are enough capturing sub expressions, this -+ // may be a back reference -+ accumulator = parseAsDecimal(octAccumulator); -+ if (accumulator < groupIndex) { -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } else { -+ addEscapedOctal(currentSeq, octAccumulator, mode); -+ } -+ } -+ } break; -+ case 410: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } -+ } break; -+ case 411: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } -+ } break; -+ case 412: -+ -+ { -+ te = p; -+ p--; -+ { -+ // Accumulator is a negative offset. -+ if (accumulator == 0) { -+ throw LocatedParseError( -+ "Numbered reference cannot be zero"); -+ } -+ if (accumulator >= groupIndex) { -+ throw LocatedParseError("Invalid reference"); -+ } -+ unsigned idx = groupIndex - accumulator; -+ currentSeq->addComponent( -+ ue2::make_unique(idx)); -+ } -+ } break; -+ case 413: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Invalid reference after \\g"); } -+ } break; -+ case 414: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError("Value in \\o{...} sequence is " -+ "non-octal or missing braces"); -+ } -+ } break; -+ case 415: -+ -+ { -+ te = p; -+ p--; -+ { addEscapedHex(currentSeq, accumulator, mode); } -+ } break; -+ case 416: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError("Value in \\x{...} sequence is " -+ "non-hex or missing }"); -+ } -+ } break; -+ case 417: -+ -+ { -+ te = p; -+ p--; -+ { -+ if (te - ts < 3) { -+ assert(te - ts == 2); -+ throw LocatedParseError(SLASH_C_ERROR); -+ } else { -+ assert(te - ts == 3); -+ addLiteral(currentSeq, decodeCtrl(ts[2]), mode); -+ } -+ } -+ } break; -+ case 418: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Malformed property"); } -+ } break; -+ case 419: -+ -+ { -+ te = p; -+ p--; -+ { throw LocatedParseError("Malformed property"); } -+ } break; -+ case 420: -+ -+ { -+ te = p; -+ p--; -+ { -+ ostringstream str; -+ str << "\\k at index " << ts - ptr << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 421: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(ts + 1 == pe); -+ ostringstream str; -+ str << "Unescaped \\ at end of input, index " -+ << ts - ptr << "."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 422: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError( -+ "Conditional subpattern unsupported"); -+ } -+ } break; -+ case 423: -+ -+ { -+ te = p; -+ p--; -+ { -+ throw LocatedParseError( -+ "Unrecognised character after (?"); -+ } -+ } break; -+ case 424: -+ -+ { -+ te = p; -+ p--; -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 425: -+ -+ { -+ te = p; -+ p--; -+ { addLiteral(currentSeq, *ts, mode); } -+ } break; -+ case 426: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ PUSH_SEQUENCE; -+ auto seq = ue2::make_unique(); -+ seq->setCaptureIndex(groupIndex++); -+ currentSeq = enterSequence(currentSeq, move(seq)); -+ } -+ } break; -+ case 427: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ assert(!currentCls); -+ assert(!inCharClass); // not reentrant -+ currentCls = getComponentClass(mode); -+ inCharClass = true; -+ inCharClassEarly = true; -+ currentClsBegin = ts; -+ { -+ cs = 836; -+ goto _again; -+ } -+ } -+ } break; -+ case 428: -+ -+ { -+ { p = ((te)) - 1; } -+ { throw LocatedParseError("Invalid reference after \\g"); } -+ } break; -+ case 429: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError("Value in \\o{...} sequence is " -+ "non-octal or missing braces"); -+ } -+ } break; -+ case 430: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError("Value in \\x{...} sequence is " -+ "non-hex or missing }"); -+ } -+ } break; -+ case 431: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ ostringstream str; -+ str << "\\k at index " << ts - ptr << " not supported."; -+ throw ParseError(str.str()); -+ } -+ } break; -+ case 432: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError( -+ "Conditional subpattern unsupported"); -+ } -+ } break; -+ case 433: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ throw LocatedParseError( -+ "Unrecognised character after (?"); -+ } -+ } break; -+ case 434: -+ -+ { -+ { p = ((te)) - 1; } -+ { -+ assert(mode.utf8); -+ throwInvalidUtf8(); -+ } -+ } break; -+ case 435: -+ -+ { -+ { p = ((te)) - 1; } -+ { addLiteral(currentSeq, *ts, mode); } -+ } break; -+ case 436: -+ -+ { -+ switch (act) { -+ case 288: { -+ { p = ((te)) - 1; } -+ // If there are enough capturing sub expressions, this -+ // may be a back reference -+ accumulator = parseAsDecimal(octAccumulator); -+ if (accumulator < groupIndex) { -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } else { -+ addEscapedOctal(currentSeq, octAccumulator, mode); -+ } -+ } break; -+ case 290: { -+ { p = ((te)) - 1; } -+ // if there are enough left parens to this point, back -+ // ref -+ if (accumulator < groupIndex) { -+ currentSeq->addComponent( -+ ue2::make_unique( -+ accumulator)); -+ } else { -+ // Otherwise, we interpret the first three digits as -+ // an octal escape, and the remaining characters -+ // stand for themselves as literals. -+ const char *s = ts; -+ unsigned int accum = 0; -+ unsigned int oct_digits = 0; -+ assert(*s == '\\'); // token starts at backslash -+ for (++s; s < te && oct_digits < 3; -+ ++oct_digits, ++s) { -+ u8 digit = *s - '0'; -+ if (digit < 8) { -+ accum = digit + accum * 8; -+ } else { -+ break; -+ } -+ } -+ -+ if (oct_digits > 0) { -+ addEscapedOctal(currentSeq, accum, mode); -+ } -+ -+ // And then the rest of the digits, if any, are -+ // literal. -+ for (; s < te; ++s) { -+ addLiteral(currentSeq, *s, mode); -+ } -+ } -+ } break; -+ case 330: { -+ { p = ((te)) - 1; } -+ addLiteral(currentSeq, *(ts + 1), mode); -+ } break; -+ } -+ } break; -+ } -+ } -+ -+ _again: -+ _acts = _regex_actions + _regex_to_state_actions[cs]; -+ _nacts = (unsigned int)*_acts++; -+ while (_nacts-- > 0) { -+ switch (*_acts++) { -+ case 23: -+ -+ { -+ ts = 0; -+ } break; -+ } -+ } -+ -+ if (cs == 0) -+ goto _out; -+ if (++p != pe) -+ goto _resume; -+ _test_eof : {} -+ if (p == eof) { -+ if (_regex_eof_trans[cs] > 0) { -+ _trans = _regex_eof_trans[cs] - 1; -+ goto _eof_trans; -+ } -+ const short *__acts = _regex_actions + _regex_eof_actions[cs]; -+ unsigned int __nacts = (unsigned int)*__acts++; -+ while (__nacts-- > 0) { -+ switch (*__acts++) { -+ case 22: -+ -+ { -+ throw LocatedParseError("Malformed property"); -+ } break; -+ } -+ } -+ } -+ -+ _out : {} -+ } -+ -+ if (p != pe && *p != '\0') { -+ // didn't make it to the end of our input, but we didn't throw a -+ // ParseError? -+ assert(0); -+ ostringstream str; -+ str << "Parse error at index " << (p - ptr) << "."; -+ throw ParseError(str.str()); -+ } -+ -+ if (currentCls) { -+ assert(inCharClass); -+ assert(currentClsBegin); -+ ostringstream oss; -+ oss << "Unterminated character class starting at index " -+ << currentClsBegin - ptr << "."; -+ throw ParseError(oss.str()); -+ } -+ -+ if (inComment) { -+ throw ParseError("Unterminated comment."); -+ } -+ -+ if (!sequences.empty()) { -+ ostringstream str; -+ str << "Missing close parenthesis for group started at index " -+ << sequences.back().seqOffset << "."; -+ throw ParseError(str.str()); -+ } -+ -+ // Unlikely, but possible -+ if (groupIndex > 65535) { -+ throw ParseError( -+ "The maximum number of capturing subexpressions is 65535."); -+ } -+ -+ // Finalize the top-level sequence, which will take care of any -+ // top-level alternation. -+ currentSeq->finalize(); -+ assert(currentSeq == rootSeq.get()); -+ -+ // Ensure that all references are valid. -+ checkReferences(*rootSeq, groupIndex, groupNames); -+ -+ return move(rootSeq); -+ } catch (LocatedParseError &error) { -+ if (ts >= ptr && ts <= pe) { -+ error.locate(ts - ptr); -+ } else { -+ error.locate(0); -+ } -+ throw; -+ } -+} -+ -+} // namespace ue2 -diff --git a/src/parser/control_verbs.cpp b/src/parser/control_verbs.cpp -new file mode 100644 -index 0000000..482004d ---- /dev/null -+++ b/src/parser/control_verbs.cpp -@@ -0,0 +1,340 @@ -+ -+/* -+ * Copyright (c) 2017, Intel Corporation -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * -+ * * Redistributions of source code must retain the above copyright notice, -+ * this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Intel Corporation nor the names of its contributors -+ * may be used to endorse or promote products derived from this software -+ * without specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+/** -+ * \file -+ * \brief Parser for control verbs that can occur at the beginning of a pattern. -+ */ -+ -+#include "parser/control_verbs.h" -+ -+#include "parser/Parser.h" -+#include "parser/parse_error.h" -+ -+#include -+#include -+ -+using namespace std; -+ -+namespace ue2 { -+ -+const char *read_control_verbs(const char *ptr, const char *end, size_t start, -+ ParseMode &mode) { -+ const char *p = ptr; -+ const char *pe = end; -+ const char *eof = pe; -+ const char *ts, *te; -+ int cs; -+ UNUSED int act; -+ -+ static const char _ControlVerbs_actions[] = { -+ 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9}; -+ -+ static const unsigned char _ControlVerbs_key_offsets[] = { -+ 0, 7, 8, 10, 12, 14, 16, 18, 20, 21, 23, 25, 27, -+ 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 55, -+ 57, 59, 61, 63, 66, 68, 70, 72, 74, 76, 79, 82, 84, -+ 86, 88, 90, 92, 94, 96, 98, 100, 102, 105, 107, 109, 111, -+ 113, 115, 117, 119, 121, 123, 125, 127, 129, 131, 133, 135, 137, -+ 139, 141, 143, 146, 148, 149, 151, 155, 157, 159, 160, 161}; -+ -+ static const char _ControlVerbs_trans_keys[] = { -+ 41, 65, 66, 67, 76, 78, 85, 41, 41, 78, 41, 89, 41, 67, 41, 82, 41, -+ 76, 41, 70, 41, 41, 83, 41, 82, 41, 95, 41, 65, 85, 41, 78, 41, 89, -+ 41, 67, 41, 78, 41, 73, 41, 67, 41, 79, 41, 68, 41, 69, 41, 82, 41, -+ 76, 41, 70, 73, 41, 77, 41, 73, 41, 84, 41, 95, 41, 77, 82, 41, 65, -+ 41, 84, 41, 67, 41, 72, 41, 61, 41, 48, 57, 41, 48, 57, 41, 69, 41, -+ 67, 41, 85, 41, 82, 41, 83, 41, 73, 41, 79, 41, 78, 41, 79, 41, 95, -+ 41, 65, 83, 41, 85, 41, 84, 41, 79, 41, 95, 41, 80, 41, 79, 41, 83, -+ 41, 83, 41, 69, 41, 83, 41, 83, 41, 84, 41, 65, 41, 82, 41, 84, 41, -+ 95, 41, 79, 41, 80, 41, 84, 41, 67, 84, 41, 80, 41, 41, 70, 41, 49, -+ 51, 56, 41, 54, 41, 50, 41, 40, 42, 0}; -+ -+ static const char _ControlVerbs_single_lengths[] = { -+ 7, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, -+ 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, -+ 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -+ 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 2, 4, 2, 2, 1, 1, 1}; -+ -+ static const char _ControlVerbs_range_lengths[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -+ -+ static const short _ControlVerbs_index_offsets[] = { -+ 0, 8, 10, 13, 16, 19, 22, 25, 28, 30, 33, 36, 39, -+ 43, 46, 49, 52, 55, 58, 61, 64, 67, 70, 73, 76, 80, -+ 83, 86, 89, 92, 96, 99, 102, 105, 108, 111, 114, 117, 120, -+ 123, 126, 129, 132, 135, 138, 141, 144, 147, 151, 154, 157, 160, -+ 163, 166, 169, 172, 175, 178, 181, 184, 187, 190, 193, 196, 199, -+ 202, 205, 208, 212, 215, 217, 220, 225, 228, 231, 233, 235}; -+ -+ static const char _ControlVerbs_indicies[] = { -+ 0, 2, 3, 4, 5, 6, 7, 1, 8, 1, 8, 9, 1, 8, 10, 1, 11, -+ 12, 1, 8, 13, 1, 8, 14, 1, 8, 15, 1, 11, 1, 8, 16, 1, 8, -+ 17, 1, 8, 18, 1, 8, 19, 20, 1, 8, 21, 1, 8, 22, 1, 8, 12, -+ 1, 8, 23, 1, 8, 24, 1, 8, 25, 1, 8, 26, 1, 8, 27, 1, 8, -+ 15, 1, 8, 28, 1, 11, 14, 1, 8, 15, 29, 1, 8, 30, 1, 8, 31, -+ 1, 8, 32, 1, 8, 33, 1, 8, 34, 35, 1, 8, 36, 1, 8, 37, 1, -+ 8, 38, 1, 8, 39, 1, 8, 40, 1, 8, 41, 1, 11, 41, 1, 8, 42, -+ 1, 8, 43, 1, 8, 44, 1, 8, 45, 1, 8, 46, 1, 8, 47, 1, 8, -+ 48, 1, 8, 39, 1, 8, 49, 1, 8, 50, 1, 8, 51, 52, 1, 8, 53, -+ 1, 8, 54, 1, 8, 55, 1, 8, 56, 1, 8, 57, 1, 8, 58, 1, 8, -+ 59, 1, 8, 60, 1, 8, 61, 1, 8, 62, 1, 8, 15, 1, 8, 63, 1, -+ 8, 64, 1, 8, 65, 1, 8, 66, 1, 8, 67, 1, 8, 68, 1, 8, 69, -+ 1, 8, 15, 1, 8, 70, 71, 1, 8, 72, 1, 73, 1, 8, 74, 1, 75, -+ 76, 77, 78, 1, 8, 15, 1, 8, 15, 1, 75, 1, 80, 79, 82, 81, 0}; -+ -+ static const char _ControlVerbs_trans_targs[] = { -+ 75, 1, 2, 9, 22, 24, 45, 67, 75, 3, 4, 75, 5, 6, 7, 8, 10, -+ 11, 12, 13, 16, 14, 15, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, -+ 30, 37, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 43, 44, 46, 47, -+ 48, 59, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 63, 64, -+ 65, 66, 68, 70, 69, 75, 71, 75, 72, 73, 74, 75, 76, 75, 0}; -+ -+ static const char _ControlVerbs_trans_actions[] = { -+ 19, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 15, 5, 17, 0}; -+ -+ static const char _ControlVerbs_to_state_actions[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}; -+ -+ static const char _ControlVerbs_from_state_actions[] = { -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0}; -+ -+ static const short _ControlVerbs_eof_trans[] = { -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 82}; -+ -+ static const int ControlVerbs_start = 75; -+ static const int ControlVerbs_first_final = 75; -+ static const int ControlVerbs_error = -1; -+ -+ static const int ControlVerbs_en_main = 75; -+ -+ { -+ cs = ControlVerbs_start; -+ ts = 0; -+ te = 0; -+ act = 0; -+ } -+ -+ try { -+ -+ { -+ int _klen; -+ unsigned int _trans; -+ const char *_acts; -+ unsigned int _nacts; -+ const char *_keys; -+ -+ if (p == pe) -+ goto _test_eof; -+ _resume: -+ _acts = -+ _ControlVerbs_actions + _ControlVerbs_from_state_actions[cs]; -+ _nacts = (unsigned int)*_acts++; -+ while (_nacts-- > 0) { -+ switch (*_acts++) { -+ case 1: { -+ ts = p; -+ } break; -+ } -+ } -+ -+ _keys = _ControlVerbs_trans_keys + _ControlVerbs_key_offsets[cs]; -+ _trans = _ControlVerbs_index_offsets[cs]; -+ -+ _klen = _ControlVerbs_single_lengths[cs]; -+ if (_klen > 0) { -+ const char *_lower = _keys; -+ const char *_mid; -+ const char *_upper = _keys + _klen - 1; -+ while (1) { -+ if (_upper < _lower) -+ break; -+ -+ _mid = _lower + ((_upper - _lower) >> 1); -+ if ((*p) < *_mid) -+ _upper = _mid - 1; -+ else if ((*p) > *_mid) -+ _lower = _mid + 1; -+ else { -+ _trans += (unsigned int)(_mid - _keys); -+ goto _match; -+ } -+ } -+ _keys += _klen; -+ _trans += _klen; -+ } -+ -+ _klen = _ControlVerbs_range_lengths[cs]; -+ if (_klen > 0) { -+ const char *_lower = _keys; -+ const char *_mid; -+ const char *_upper = _keys + (_klen << 1) - 2; -+ while (1) { -+ if (_upper < _lower) -+ break; -+ -+ _mid = _lower + (((_upper - _lower) >> 1) & ~1); -+ if ((*p) < _mid[0]) -+ _upper = _mid - 2; -+ else if ((*p) > _mid[1]) -+ _lower = _mid + 2; -+ else { -+ _trans += (unsigned int)((_mid - _keys) >> 1); -+ goto _match; -+ } -+ } -+ _trans += _klen; -+ } -+ -+ _match: -+ _trans = _ControlVerbs_indicies[_trans]; -+ _eof_trans: -+ cs = _ControlVerbs_trans_targs[_trans]; -+ -+ if (_ControlVerbs_trans_actions[_trans] == 0) -+ goto _again; -+ -+ _acts = _ControlVerbs_actions + _ControlVerbs_trans_actions[_trans]; -+ _nacts = (unsigned int)*_acts++; -+ while (_nacts-- > 0) { -+ switch (*_acts++) { -+ case 2: { -+ te = p + 1; -+ } break; -+ case 3: { -+ te = p + 1; -+ { mode.utf8 = true; } -+ } break; -+ case 4: { -+ te = p + 1; -+ { mode.ucp = true; } -+ } break; -+ case 5: { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "Unsupported control verb " -+ << string(ts, te - ts); -+ throw LocatedParseError(str.str()); -+ } -+ } break; -+ case 6: { -+ te = p + 1; -+ { -+ ostringstream str; -+ str << "Unknown control verb " << string(ts, te - ts); -+ throw LocatedParseError(str.str()); -+ } -+ } break; -+ case 7: { -+ te = p + 1; -+ { -+ p--; -+ { -+ p++; -+ goto _out; -+ } -+ } -+ } break; -+ case 8: { -+ te = p; -+ p--; -+ { -+ p--; -+ { -+ p++; -+ goto _out; -+ } -+ } -+ } break; -+ case 9: { -+ { p = ((te)) - 1; } -+ { -+ p--; -+ { -+ p++; -+ goto _out; -+ } -+ } -+ } break; -+ } -+ } -+ -+ _again: -+ _acts = _ControlVerbs_actions + _ControlVerbs_to_state_actions[cs]; -+ _nacts = (unsigned int)*_acts++; -+ while (_nacts-- > 0) { -+ switch (*_acts++) { -+ case 0: { -+ ts = 0; -+ } break; -+ } -+ } -+ -+ if (++p != pe) -+ goto _resume; -+ _test_eof : {} -+ if (p == eof) { -+ if (_ControlVerbs_eof_trans[cs] > 0) { -+ _trans = _ControlVerbs_eof_trans[cs] - 1; -+ goto _eof_trans; -+ } -+ } -+ -+ _out : {} -+ } -+ -+ } catch (LocatedParseError &error) { -+ if (ts >= ptr && ts <= pe) { -+ error.locate(ts - ptr + start); -+ } else { -+ error.locate(0); -+ } -+ throw; -+ } -+ -+ return p; -+} -+ -+} // namespace ue2 -diff --git a/src/rose/counting_miracle.h b/src/rose/counting_miracle.h -index 976208b..4456679 100644 ---- a/src/rose/counting_miracle.h -+++ b/src/rose/counting_miracle.h -@@ -94,7 +94,7 @@ u32 roseCountingMiracleScanShufti(m128 mask_lo, m128 mask_hi, u8 poison, - u32 count = *count_inout; - - const m128 zeroes = zeroes128(); -- const m128 low4bits = _mm_set1_epi8(0xf); -+ const m128 low4bits = set16x8(0xf); - - for (; d + 16 <= d_end; d_end -= 16) { - m128 data = loadu128(d_end - 16); -diff --git a/src/util/arch.h b/src/util/arch.h -index 985fec6..fe4a910 100644 ---- a/src/util/arch.h -+++ b/src/util/arch.h -@@ -61,6 +61,10 @@ - #define HAVE_AVX512VBMI - #endif - -+#if defined(__aarch64__) -+#define HAVE_NEON -+#endif -+ - /* - * ICC and MSVC don't break out POPCNT or BMI/2 as separate pre-def macros - */ -@@ -87,4 +91,11 @@ - #define NO_ASM - #endif - -+/* -+ * AARCH64 uses a different form of inline asm -+ */ -+#if defined(__aarch64__) -+#define NO_ASM -+#endif -+ - #endif // UTIL_ARCH_H_ -diff --git a/src/util/cpuid_flags.c b/src/util/cpuid_flags.c -index c00ce58..e0f6368 100644 ---- a/src/util/cpuid_flags.c -+++ b/src/util/cpuid_flags.c -@@ -39,7 +39,7 @@ - - u64a cpuid_flags(void) { - u64a cap = 0; -- -+#if defined(__X86_64__) - if (check_avx2()) { - DEBUG_PRINTF("AVX2 enabled\n"); - cap |= HS_CPU_FEATURES_AVX2; -@@ -68,7 +68,7 @@ u64a cpuid_flags(void) { - (defined(FAT_RUNTIME) && !defined(BUILD_AVX512VBMI)) - cap &= ~HS_CPU_FEATURES_AVX512VBMI; - #endif -- -+#endif - return cap; - } - -@@ -78,6 +78,7 @@ struct family_id { - u32 tune; - }; - -+#if defined(__X86_64__) - /* from table 35-1 of the Intel 64 and IA32 Arch. Software Developer's Manual - * and "Intel Architecture and Processor Identification With CPUID Model and - * Family Numbers" */ -@@ -121,6 +122,7 @@ static const struct family_id known_microarch[] = { - { 0x6, 0x6C, HS_TUNE_FAMILY_ICX }, /* Icelake Xeon */ - - }; -+#endif - - #ifdef DUMP_SUPPORT - static UNUSED -@@ -144,6 +146,7 @@ const char *dumpTune(u32 tune) { - #endif - - u32 cpuid_tune(void) { -+#if defined(__X86_64__) - unsigned int eax, ebx, ecx, edx; - - cpuid(1, 0, &eax, &ebx, &ecx, &edx); -@@ -171,6 +174,6 @@ u32 cpuid_tune(void) { - DEBUG_PRINTF("found tune flag %s\n", dumpTune(tune) ); - return tune; - } -- -+#endif - return HS_TUNE_FAMILY_GENERIC; - } -diff --git a/src/util/cpuid_flags.h b/src/util/cpuid_flags.h -index 527c6d5..3125bd1 100644 ---- a/src/util/cpuid_flags.h -+++ b/src/util/cpuid_flags.h -@@ -32,7 +32,9 @@ - #include "ue2common.h" - - #if !defined(_WIN32) && !defined(CPUID_H_) -+#if defined(__x86_64__) - #include -+#endif - /* system header doesn't have a header guard */ - #define CPUID_H_ - #endif -diff --git a/src/util/cpuid_inline.h b/src/util/cpuid_inline.h -index b7b4245..b228c1d 100644 ---- a/src/util/cpuid_inline.h -+++ b/src/util/cpuid_inline.h -@@ -32,17 +32,20 @@ - #include "ue2common.h" - #include "cpuid_flags.h" - -+#if defined(__x86_64__) || defined(_M_X64) - #if !defined(_WIN32) && !defined(CPUID_H_) - #include - /* system header doesn't have a header guard */ - #define CPUID_H_ - #endif -+#endif - - #ifdef __cplusplus - extern "C" - { - #endif - -+#if defined(__x86_64__) || defined(_M_X64) - static inline - void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, - unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { -@@ -57,6 +60,7 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, - *edx = a[3]; - #endif - } -+#endif - - // ECX - #define CPUID_SSE3 (1 << 0) -@@ -93,11 +97,12 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, - #define CPUID_XCR0_AVX512 \ - (CPUID_XCR0_OPMASK | CPUID_XCR0_ZMM_Hi256 | CPUID_XCR0_Hi16_ZMM) - -+#if defined(__x86_64__) - static inline - u64a xgetbv(u32 op) { - #if defined(_WIN32) || defined(__INTEL_COMPILER) - return _xgetbv(op); --#else -+#elif defined(__x86_64__) - u32 a, d; - __asm__ volatile ( - "xgetbv\n" -@@ -252,6 +257,16 @@ int check_popcnt(void) { - cpuid(1, 0, &eax, &ebx, &ecx, &edx); - return !!(ecx & CPUID_POPCNT); - } -+#endif //__x86_64__ -+ -+static inline -+int check_neon(void) { -+#if defined(__aarch64__) -+ return 1; -+#else -+ return 0; -+#endif -+} - - #ifdef __cplusplus - } /* extern "C" */ -diff --git a/src/util/intrinsics.h b/src/util/intrinsics.h -index edc4f6e..ece3b1a 100644 ---- a/src/util/intrinsics.h -+++ b/src/util/intrinsics.h -@@ -55,10 +55,22 @@ - # endif - #endif - -+#ifdef __cplusplus -+# if defined(HAVE_CXX_ARM_NEON_H) -+# define USE_ARM_NEON_H -+# endif -+#else // C -+# if defined(HAVE_C_ARM_NEON_H) -+# define USE_ARM_NEON_H -+# endif -+#endif -+ - #if defined(USE_X86INTRIN_H) - #include - #elif defined(USE_INTRIN_H) - #include -+#elif defined(USE_ARM_NEON_H) -+#include - #else - #error no intrinsics file - #endif -diff --git a/src/util/popcount.h b/src/util/popcount.h -index eb08f6b..7d794d1 100644 ---- a/src/util/popcount.h -+++ b/src/util/popcount.h -@@ -41,6 +41,8 @@ u32 popcount32(u32 x) { - #if defined(HAVE_POPCOUNT_INSTR) - // Single-instruction builtin. - return _mm_popcnt_u32(x); -+#elif defined(HAVE_NEON) -+ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x))); - #else - // Fast branch-free version from bit-twiddling hacks as older Intel - // processors do not have a POPCNT instruction. -@@ -63,7 +65,9 @@ u32 popcount64(u64a x) { - x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); - x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; - return (x * 0x0101010101010101) >> 56; --# endif -+#endif -+#elif defined(HAVE_NEON) -+ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x))); - #else - // Synthesise from two 32-bit cases. - return popcount32(x >> 32) + popcount32(x); -diff --git a/src/util/simd_arm.h b/src/util/simd_arm.h -new file mode 100644 -index 0000000..cce119f ---- /dev/null -+++ b/src/util/simd_arm.h -@@ -0,0 +1,1069 @@ -+/* -+ * Copyright (c) 2015-2017, Intel Corporation -+ * 2020.01 - Use the neon instruction to implement the function of 128-bit operation. -+ * Huawei Technologies Co., Ltd. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * -+ * * Redistributions of source code must retain the above copyright notice, -+ * this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Intel Corporation nor the names of its contributors -+ * may be used to endorse or promote products derived from this software -+ * without specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+/** \file -+ * \brief SIMD types and primitive operations. -+ */ -+ -+#ifndef SIMD_ARM -+#define SIMD_ARM -+ -+#include "config.h" -+#include "simd_types.h" -+#include "ue2common.h" -+#include "unaligned.h" -+#include "util/arch.h" -+#include "util/intrinsics.h" -+ -+#include // for memcpy -+ -+// Define a common assume_aligned using an appropriate compiler built-in, if -+// it's available. Note that we need to handle C or C++ compilation. -+#ifdef __cplusplus -+#ifdef HAVE_CXX_BUILTIN_ASSUME_ALIGNED -+#define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) -+#endif -+#else -+#ifdef HAVE_CC_BUILTIN_ASSUME_ALIGNED -+#define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) -+#endif -+#endif -+ -+// Fallback to identity case. -+#ifndef assume_aligned -+#define assume_aligned(x, y) (x) -+#endif -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+extern const char vbs_mask_data[]; -+#ifdef __cplusplus -+} -+#endif -+ -+/* -+** extend 4.8.5 neon inline assembly functions -+*/ -+__extension__ static __inline uint64x2_t __attribute__((__always_inline__)) -+vmvnq_u64(uint64x2_t a) { -+ uint64x2_t result; -+ __asm__("mvn %0.16b,%1.16b" : "=w"(result) : "w"(a) : /* No clobbers */); -+ return result; -+} -+ -+#pragma GCC diagnostic push -+#pragma GCC diagnostic ignored "-Wshadow" -+ -+static really_inline m128 ones128(void) { -+ m128 result; -+ result.vect_s32 = vdupq_n_s32(0xFFFFFFFF); -+ return result; -+} -+ -+static really_inline m128 zeroes128(void) { -+ m128 result; -+ result.vect_s32 = vdupq_n_s32(0x0); -+ return result; -+} -+ -+/** \brief Return 1 if a and b are different otherwise 0 */ -+static really_inline int diff128(m128 a, m128 b) { -+ return !!vaddlvq_s16(veorq_s16(a.vect_s16, b.vect_s16)); -+} -+ -+static really_inline int isnonzero128(m128 a) { -+ return !!diff128(a, zeroes128()); -+} -+ -+/** -+ * "Rich" version of diff128(). Takes two vectors a and b and returns a 4-bit -+ * mask indicating which 32-bit words contain differences. -+ */ -+static really_inline u32 diffrich128(m128 a, m128 b) { -+ m128 tmp; -+ tmp.vect_u32 = vmvnq_u32(vceqq_u32(a.vect_u32, b.vect_u32)); -+ return ((vgetq_lane_u32(tmp.vect_u32, 3) & 0x8) | -+ (vgetq_lane_u32(tmp.vect_u32, 2) & 0x4) | -+ (vgetq_lane_u32(tmp.vect_u32, 1) & 0x2) | -+ (vgetq_lane_u32(tmp.vect_u32, 0) & 0x1)); -+} -+ -+/** -+ * "Rich" version of diff128(), 64-bit variant. Takes two vectors a and b and -+ * returns a 4-bit mask indicating which 64-bit words contain differences. -+ */ -+static really_inline u32 diffrich64_128(m128 a, m128 b) { -+ m128 tmp; -+ tmp.vect_u64 = vmvnq_u64(vceqq_u64(a.vect_u64, b.vect_u64)); -+ return (u32)((vgetq_lane_u64(tmp.vect_u64, 1) & 0x4) | -+ (vgetq_lane_u64(tmp.vect_u64, 0) & 0x1)); -+} -+ -+static really_really_inline m128 lshift64_m128(m128 a, unsigned b) { -+ assert(b <= 63); -+ m128 result; -+ result.vect_s64 = vshlq_n_s64(a.vect_s64, b); -+ return result; -+} -+ -+static really_really_inline m128 rshift64_m128(m128 a, int imm8) { -+ assert(imm8 >= 0 && imm8 <= 63); -+ if (unlikely(imm8 == 0)) { -+ return a; -+ } -+ m128 result; -+ result.vect_u64 = vshrq_n_u64(a.vect_u64, imm8); -+ return result; -+} -+ -+static really_really_inline m128 eq128(m128 a, m128 b) { -+ m128 result; -+ result.vect_u8 = vceqq_s8(a.vect_s8, b.vect_s8); -+ return result; -+} -+ -+static really_really_inline u32 movemask128(m128 a) { -+ m128 result; -+ result.vect_u8 = vshrq_n_u8(a.vect_u8, 7); -+ result.vect_u16 = vsraq_n_u16(result.vect_u16, result.vect_u16, 7); -+ result.vect_u32 = vsraq_n_u32(result.vect_u32, result.vect_u32, 14); -+ result.vect_u64 = vsraq_n_u64(result.vect_u64, result.vect_u64, 28); -+ return (u32)(vgetq_lane_u8(result.vect_u8, 0) | -+ ((u32)vgetq_lane_u8(result.vect_u8, 8) << 8)); -+} -+ -+static really_really_inline m128 rshiftbyte_m128(m128 a, int imm8) { -+ assert(imm8 >= 0 && imm8 <= 15); -+ m128 result; -+ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0), imm8); -+ return result; -+} -+ -+static really_really_inline m128 lshiftbyte_m128(m128 a, int imm8) { -+ assert(imm8 >= 0 && imm8 <= 15); -+ m128 result; -+ if (unlikely(imm8 == 0)) { -+ return a; -+ } -+ result.vect_s8 = vextq_s8(vdupq_n_s8(0), a.vect_s8, (16 - imm8)); -+ return result; -+} -+ -+static really_inline m128 set16x8(u8 c) { -+ m128 result; -+ result.vect_s8 = vdupq_n_s8(c); -+ return result; -+} -+ -+static really_inline m128 set4x32(u32 c) { -+ m128 result; -+ result.vect_s32 = vdupq_n_s32(c); -+ return result; -+} -+ -+static really_inline m128 set2x64(u64a c) { -+ m128 result; -+ result.vect_u64 = vdupq_n_u64(c); -+ return result; -+} -+ -+static really_inline u32 movd(const m128 in) { -+ u32 result; -+ result = vgetq_lane_u32(in.vect_u32, 0); -+ return result; -+} -+ -+static really_inline u64a movq(const m128 in) { -+ return vgetq_lane_u64(in.vect_u64, 0); -+} -+ -+/* another form of movq */ -+static really_inline m128 load_m128_from_u64a(const u64a *p) { -+ m128 result; -+ __asm__ __volatile__("ldr %d0, %1 \n\t" -+ : "=w"(result) -+ : "Utv"(*p) -+ : /* No clobbers */ -+ ); -+ return result; -+} -+ -+/*The x86 platform does not perform the lower 2 bit operation. -+If the value of imm exceeds 2 bit, a compilation error occurs.*/ -+static really_inline u32 extract32from128(m128 a, int imm) { -+ return vgetq_lane_s32(a.vect_s32, imm & 0x0003); -+} -+ -+/*The x86 platform does not perform the lower 1 bit operation. -+If the value of imm exceeds 1 bit, a compilation error occurs.*/ -+static really_inline u64a extract64from128(m128 a, int imm) { -+ return vgetq_lane_s64(a.vect_s64, imm & 0x0001); -+} -+ -+#define extractlow64from256(a) movq(a.lo) -+#define extractlow32from256(a) movd(a.lo) -+ -+/*The x86 platform does not perform the lower 2 bit operation. -+If the value of imm exceeds 2 bit, a compilation error occurs.*/ -+static really_inline u32 extract32from256(m256 a, int imm) { -+ return vgetq_lane_s32((imm >> 2) ? a.hi.vect_s32 : a.lo.vect_s32, -+ imm & 0x0003); -+} -+ -+/*The x86 platform does not perform the lower 1 bit operation. -+If the value of imm exceeds 1 bit, a compilation error occurs.*/ -+static really_inline u64a extract64from256(m256 a, int imm) { -+ return vgetq_lane_s64((imm >> 1) ? a.hi.vect_s64 : a.lo.vect_s64, -+ imm & 0x0001); -+} -+ -+static really_inline m128 and128(m128 a, m128 b) { -+ m128 result; -+ result.vect_s32 = vandq_s32(a.vect_s32, b.vect_s32); -+ return result; -+} -+ -+static really_inline m128 not128(m128 a) { -+ m128 result; -+ result.vect_s32 = vmvnq_s32(a.vect_s32); -+ return result; -+} -+ -+static really_inline m128 xor128(m128 a, m128 b) { -+ m128 result; -+ result.vect_s32 = veorq_s32(a.vect_s32, b.vect_s32); -+ return result; -+} -+ -+static really_inline m128 or128(m128 a, m128 b) { -+ m128 result; -+ result.vect_s32 = vorrq_s32(a.vect_s32, b.vect_s32); -+ return result; -+} -+ -+static really_inline m128 andnot128(m128 a, m128 b) { -+ m128 result; -+ result.vect_s32 = vbicq_s32(b.vect_s32, a.vect_s32); -+ return result; -+} -+ -+// aligned load -+static really_inline m128 load128(const void *ptr) { -+ assert(ISALIGNED_N(ptr, alignof(m128))); -+ ptr = assume_aligned(ptr, 16); -+ m128 result; -+ result.vect_s32 = vld1q_s32((const int32_t *)ptr); -+ return result; -+} -+ -+// aligned store -+static really_inline void store128(void *ptr, m128 a) { -+ assert(ISALIGNED_N(ptr, alignof(m128))); -+ ptr = assume_aligned(ptr, 16); -+ *(m128 *)ptr = a; -+} -+ -+// unaligned load -+static really_inline m128 loadu128(const void *ptr) { -+ m128 result; -+ result.vect_s32 = vld1q_s32((const int32_t *)ptr); -+ return result; -+} -+ -+// unaligned store -+static really_inline void storeu128(void *ptr, m128 a) { -+ vst1q_s32((int32_t *)ptr, a.vect_s32); -+} -+ -+// packed unaligned store of first N bytes -+static really_inline void storebytes128(void *ptr, m128 a, unsigned int n) { -+ assert(n <= sizeof(a)); -+ memcpy(ptr, &a, n); -+} -+ -+// packed unaligned load of first N bytes, pad with zero -+static really_inline m128 loadbytes128(const void *ptr, unsigned int n) { -+ m128 a = zeroes128(); -+ assert(n <= sizeof(a)); -+ memcpy(&a, ptr, n); -+ return a; -+} -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+extern const u8 simd_onebit_masks[]; -+#ifdef __cplusplus -+} -+#endif -+ -+static really_inline m128 mask1bit128(unsigned int n) { -+ assert(n < sizeof(m128) * 8); -+ u32 mask_idx = ((n % 8) * 64) + 95; -+ mask_idx -= n / 8; -+ return loadu128(&simd_onebit_masks[mask_idx]); -+} -+ -+// switches on bit N in the given vector. -+static really_inline void setbit128(m128 *ptr, unsigned int n) { -+ *ptr = or128(mask1bit128(n), *ptr); -+} -+ -+// switches off bit N in the given vector. -+static really_inline void clearbit128(m128 *ptr, unsigned int n) { -+ *ptr = andnot128(mask1bit128(n), *ptr); -+} -+ -+// tests bit N in the given vector. -+static really_inline char testbit128(m128 val, unsigned int n) { -+ const m128 mask = mask1bit128(n); -+ return isnonzero128(and128(mask, val)); -+} -+ -+// offset must be an immediate -+/*The x86 platform does not perform the lower 8 bit operation. -+If the value of imm exceeds 8 bit, a compilation error occurs.*/ -+static really_inline m128 palignr(m128 a, m128 b, int count) { -+ m128 result; -+ count = count & 0xff; -+ if (likely(count < 16)) { -+ result.vect_s8 = vextq_s8(b.vect_s8, a.vect_s8, count); -+ } else if (count < 32) { -+ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0x0), count - 16); -+ } else { -+ result.vect_s32 = vdupq_n_s32(0); -+ } -+ return result; -+} -+ -+static really_inline m128 pshufb_m128(m128 a, m128 b) { -+ m128 result; -+ __asm__ __volatile__("movi v3.16b, 0x8f \n\t" -+ "and v3.16b, v3.16b, %2.16b \n\t" -+ "tbl %0.16b, {%1.16b}, v3.16b \n\t" -+ : "=w"(result) -+ : "w"(a), "w"(b) -+ : "v3"); -+ return result; -+} -+ -+static really_inline m256 pshufb_m256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = pshufb_m128(a.lo, b.lo); -+ rv.hi = pshufb_m128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m128 variable_byte_shift_m128(m128 in, s32 amount) { -+ assert(amount >= -16 && amount <= 16); -+ m128 shift_mask = loadu128(vbs_mask_data + 16 - amount); -+ return pshufb_m128(in, shift_mask); -+} -+ -+static really_inline m128 max_u8_m128(m128 a, m128 b) { -+ m128 result; -+ result.vect_u8 = vmaxq_u8(a.vect_u8, b.vect_u8); -+ return result; -+} -+ -+static really_inline m128 min_u8_m128(m128 a, m128 b) { -+ m128 result; -+ result.vect_u8 = vminq_u8(a.vect_u8, b.vect_u8); -+ return result; -+} -+ -+static really_inline m128 sadd_u8_m128(m128 a, m128 b) { -+ m128 result; -+ result.vect_u8 = vqaddq_u8(a.vect_u8, b.vect_u8); -+ return result; -+} -+ -+static really_inline m128 sub_u8_m128(m128 a, m128 b) { -+ m128 result; -+ result.vect_u8 = vsubq_u8(a.vect_u8, b.vect_u8); -+ return result; -+} -+ -+static really_inline m128 set64x2(int64_t hi, int64_t lo) { -+ m128 result; -+ result.vect_s64 = vsetq_lane_s64(hi, vdupq_n_s64(lo), 1); -+ return result; -+} -+ -+static really_inline m128 set32x4(int i3, int i2, int i1, int i0) { -+ m128 result; -+ result.vect_s32 = vsetq_lane_s32( -+ i3, vsetq_lane_s32(i2, vsetq_lane_s32(i1, vdupq_n_s32(i0), 1), 2), 3); -+ return result; -+} -+ -+/**** -+ **** 256-bit Primitives -+ ****/ -+ -+static really_really_inline m256 lshift64_m256(m256 a, int b) { -+ m256 rv = a; -+ rv.lo = lshift64_m128(rv.lo, b); -+ rv.hi = lshift64_m128(rv.hi, b); -+ return rv; -+} -+ -+static really_inline m256 rshift64_m256(m256 a, int b) { -+ m256 rv = a; -+ rv.lo = rshift64_m128(rv.lo, b); -+ rv.hi = rshift64_m128(rv.hi, b); -+ return rv; -+} -+static really_inline m256 set32x8(u32 in) { -+ m256 rv; -+ rv.lo = set16x8((u8)in); -+ rv.hi = rv.lo; -+ return rv; -+} -+ -+static really_inline m256 eq256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = eq128(a.lo, b.lo); -+ rv.hi = eq128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline u32 movemask256(m256 a) { -+ u32 lo_mask = movemask128(a.lo); -+ u32 hi_mask = movemask128(a.hi); -+ return lo_mask | (hi_mask << 16); -+} -+ -+static really_inline m256 set2x128(m128 a) { -+ m256 rv = {a, a}; -+ return rv; -+} -+ -+static really_inline m256 zeroes256(void) { -+ m256 rv = {zeroes128(), zeroes128()}; -+ return rv; -+} -+ -+static really_inline m256 ones256(void) { -+ m256 rv = {ones128(), ones128()}; -+ return rv; -+} -+ -+static really_inline m256 and256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = and128(a.lo, b.lo); -+ rv.hi = and128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m256 or256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = or128(a.lo, b.lo); -+ rv.hi = or128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m256 xor256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = xor128(a.lo, b.lo); -+ rv.hi = xor128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m256 not256(m256 a) { -+ m256 rv; -+ rv.lo = not128(a.lo); -+ rv.hi = not128(a.hi); -+ return rv; -+} -+ -+static really_inline m256 andnot256(m256 a, m256 b) { -+ m256 rv; -+ rv.lo = andnot128(a.lo, b.lo); -+ rv.hi = andnot128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline int diff256(m256 a, m256 b) { -+ return diff128(a.lo, b.lo) || diff128(a.hi, b.hi); -+} -+ -+static really_inline int isnonzero256(m256 a) { -+ return isnonzero128(or128(a.lo, a.hi)); -+} -+ -+/** -+ * "Rich" version of diff256(). Takes two vectors a and b and returns an 8-bit -+ * mask indicating which 32-bit words contain differences. -+ */ -+static really_inline u32 diffrich256(m256 a, m256 b) { -+ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32); -+ uint32x4_t y = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32); -+ uint8x8_t lo = vqmovn_u16(vcombine_u16(vqmovn_u32(x), vqmovn_u32(y))); -+ -+ static const int8_t __attribute__((aligned(16))) -+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; -+ uint8x8_t mask_and = vdup_n_u8(0x80); -+ int8x8_t mask_shift = vld1_s8(xr); -+ -+ lo = vand_u8(lo, mask_and); -+ lo = vshl_u8(lo, mask_shift); -+ -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ -+ return ~(lo[0] & 0xFF) & 0xff; -+} -+ -+/** -+ * "Rich" version of diff256(), 64-bit variant. Takes two vectors a and b and -+ * returns an 8-bit mask indicating which 64-bit words contain differences. -+ */ -+static really_inline u32 diffrich64_256(m256 a, m256 b) { -+ u32 d = diffrich256(a, b); -+ return (d | (d >> 1)) & 0x55555555; -+} -+ -+// aligned load -+static really_inline m256 load256(const void *ptr) { -+ assert(ISALIGNED_N(ptr, alignof(m256))); -+ m256 rv = {load128(ptr), load128((const char *)ptr + 16)}; -+ return rv; -+} -+ -+// aligned load of 128-bit value to low and high part of 256-bit value -+static really_inline m256 load2x128(const void *ptr) { -+ assert(ISALIGNED_N(ptr, alignof(m128))); -+ m256 rv; -+ rv.hi = rv.lo = load128(ptr); -+ return rv; -+} -+ -+static really_inline m256 loadu2x128(const void *ptr) { -+ return set2x128(loadu128(ptr)); -+} -+ -+// aligned store -+static really_inline void store256(void *ptr, m256 a) { -+ assert(ISALIGNED_N(ptr, alignof(m256))); -+ ptr = assume_aligned(ptr, 16); -+ *(m256 *)ptr = a; -+} -+ -+// unaligned load -+static really_inline m256 loadu256(const void *ptr) { -+ m256 rv = {loadu128(ptr), loadu128((const char *)ptr + 16)}; -+ return rv; -+} -+ -+// unaligned store -+static really_inline void storeu256(void *ptr, m256 a) { -+ storeu128(ptr, a.lo); -+ storeu128((char *)ptr + 16, a.hi); -+} -+ -+// packed unaligned store of first N bytes -+static really_inline void storebytes256(void *ptr, m256 a, unsigned int n) { -+ assert(n <= sizeof(a)); -+ memcpy(ptr, &a, n); -+} -+ -+// packed unaligned load of first N bytes, pad with zero -+static really_inline m256 loadbytes256(const void *ptr, unsigned int n) { -+ m256 a = zeroes256(); -+ assert(n <= sizeof(a)); -+ memcpy(&a, ptr, n); -+ return a; -+} -+ -+static really_inline m256 mask1bit256(unsigned int n) { -+ assert(n < sizeof(m256) * 8); -+ u32 mask_idx = ((n % 8) * 64) + 95; -+ mask_idx -= n / 8; -+ return loadu256(&simd_onebit_masks[mask_idx]); -+} -+ -+static really_inline m256 set64x4(u64a hi_1, u64a hi_0, u64a lo_1, u64a lo_0) { -+ m256 rv; -+ rv.hi = set64x2(hi_1, hi_0); -+ rv.lo = set64x2(lo_1, lo_0); -+ return rv; -+} -+ -+// switches on bit N in the given vector. -+static really_inline void setbit256(m256 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo; -+ } else { -+ sub = &ptr->hi; -+ n -= 128; -+ } -+ setbit128(sub, n); -+} -+ -+// switches off bit N in the given vector. -+static really_inline void clearbit256(m256 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo; -+ } else { -+ sub = &ptr->hi; -+ n -= 128; -+ } -+ clearbit128(sub, n); -+} -+ -+// tests bit N in the given vector. -+static really_inline char testbit256(m256 val, unsigned int n) { -+ assert(n < sizeof(val) * 8); -+ m128 sub; -+ if (n < 128) { -+ sub = val.lo; -+ } else { -+ sub = val.hi; -+ n -= 128; -+ } -+ return testbit128(sub, n); -+} -+ -+static really_really_inline m128 movdq_hi(m256 x) { return x.hi; } -+ -+static really_really_inline m128 movdq_lo(m256 x) { return x.lo; } -+ -+static really_inline m256 combine2x128(m128 hi, m128 lo) { -+ m256 rv = {lo, hi}; -+ return rv; -+} -+ -+/**** -+ **** 384-bit Primitives -+ ****/ -+ -+static really_inline m384 and384(m384 a, m384 b) { -+ m384 rv; -+ rv.lo = and128(a.lo, b.lo); -+ rv.mid = and128(a.mid, b.mid); -+ rv.hi = and128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m384 or384(m384 a, m384 b) { -+ m384 rv; -+ rv.lo = or128(a.lo, b.lo); -+ rv.mid = or128(a.mid, b.mid); -+ rv.hi = or128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m384 xor384(m384 a, m384 b) { -+ m384 rv; -+ rv.lo = xor128(a.lo, b.lo); -+ rv.mid = xor128(a.mid, b.mid); -+ rv.hi = xor128(a.hi, b.hi); -+ return rv; -+} -+static really_inline m384 not384(m384 a) { -+ m384 rv; -+ rv.lo = not128(a.lo); -+ rv.mid = not128(a.mid); -+ rv.hi = not128(a.hi); -+ return rv; -+} -+static really_inline m384 andnot384(m384 a, m384 b) { -+ m384 rv; -+ rv.lo = andnot128(a.lo, b.lo); -+ rv.mid = andnot128(a.mid, b.mid); -+ rv.hi = andnot128(a.hi, b.hi); -+ return rv; -+} -+ -+static really_really_inline m384 lshift64_m384(m384 a, unsigned b) { -+ m384 rv; -+ rv.lo = lshift64_m128(a.lo, b); -+ rv.mid = lshift64_m128(a.mid, b); -+ rv.hi = lshift64_m128(a.hi, b); -+ return rv; -+} -+ -+static really_inline m384 zeroes384(void) { -+ m384 rv = {zeroes128(), zeroes128(), zeroes128()}; -+ return rv; -+} -+ -+static really_inline m384 ones384(void) { -+ m384 rv = {ones128(), ones128(), ones128()}; -+ return rv; -+} -+ -+static really_inline int diff384(m384 a, m384 b) { -+ return diff128(a.lo, b.lo) || diff128(a.mid, b.mid) || diff128(a.hi, b.hi); -+} -+ -+static really_inline int isnonzero384(m384 a) { -+ return isnonzero128(or128(or128(a.lo, a.mid), a.hi)); -+} -+ -+/** -+ * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit -+ * mask indicating which 32-bit words contain differences. -+ */ -+static really_inline u32 diffrich384(m384 a, m384 b) { -+ m128 z = zeroes128(); -+ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32); -+ uint32x4_t y = vceqq_s32(a.mid.vect_s32, b.mid.vect_s32); -+ uint32x4_t w = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32); -+ -+ uint16x8_t q = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y)); -+ uint16x8_t p = vcombine_u16(vqmovn_u32(w), vqmovn_u32(z.vect_u32)); -+ -+ uint8x16_t input = vcombine_u8(vqmovn_u16(q), vqmovn_u16(p)); -+ -+ static const int8_t __attribute__((aligned(16))) -+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; -+ uint8x8_t mask_and = vdup_n_u8(0x80); -+ int8x8_t mask_shift = vld1_s8(xr); -+ -+ uint8x8_t lo = vget_low_u8(input); -+ uint8x8_t hi = vget_high_u8(input); -+ -+ lo = vand_u8(lo, mask_and); -+ lo = vshl_u8(lo, mask_shift); -+ -+ hi = vand_u8(hi, mask_and); -+ hi = vshl_u8(hi, mask_shift); -+ -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ -+ hi = vpadd_u8(hi, hi); -+ hi = vpadd_u8(hi, hi); -+ hi = vpadd_u8(hi, hi); -+ -+ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xfff; -+} -+ -+/** -+ * "Rich" version of diff384(), 64-bit variant. Takes two vectors a and b and -+ * returns a 12-bit mask indicating which 64-bit words contain differences. -+ */ -+static really_inline u32 diffrich64_384(m384 a, m384 b) { -+ u32 d = diffrich384(a, b); -+ return (d | (d >> 1)) & 0x55555555; -+} -+ -+// aligned load -+static really_inline m384 load384(const void *ptr) { -+ assert(ISALIGNED_16(ptr)); -+ m384 rv = {load128(ptr), load128((const char *)ptr + 16), -+ load128((const char *)ptr + 32)}; -+ return rv; -+} -+ -+// aligned store -+static really_inline void store384(void *ptr, m384 a) { -+ assert(ISALIGNED_16(ptr)); -+ ptr = assume_aligned(ptr, 16); -+ *(m384 *)ptr = a; -+} -+ -+// unaligned load -+static really_inline m384 loadu384(const void *ptr) { -+ m384 rv = {loadu128(ptr), loadu128((const char *)ptr + 16), -+ loadu128((const char *)ptr + 32)}; -+ return rv; -+} -+ -+// packed unaligned store of first N bytes -+static really_inline void storebytes384(void *ptr, m384 a, unsigned int n) { -+ assert(n <= sizeof(a)); -+ memcpy(ptr, &a, n); -+} -+ -+// packed unaligned load of first N bytes, pad with zero -+static really_inline m384 loadbytes384(const void *ptr, unsigned int n) { -+ m384 a = zeroes384(); -+ assert(n <= sizeof(a)); -+ memcpy(&a, ptr, n); -+ return a; -+} -+ -+// switches on bit N in the given vector. -+static really_inline void setbit384(m384 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo; -+ } else if (n < 256) { -+ sub = &ptr->mid; -+ } else { -+ sub = &ptr->hi; -+ } -+ setbit128(sub, n % 128); -+} -+ -+// switches off bit N in the given vector. -+static really_inline void clearbit384(m384 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo; -+ } else if (n < 256) { -+ sub = &ptr->mid; -+ } else { -+ sub = &ptr->hi; -+ } -+ clearbit128(sub, n % 128); -+} -+ -+// tests bit N in the given vector. -+static really_inline char testbit384(m384 val, unsigned int n) { -+ assert(n < sizeof(val) * 8); -+ m128 sub; -+ if (n < 128) { -+ sub = val.lo; -+ } else if (n < 256) { -+ sub = val.mid; -+ } else { -+ sub = val.hi; -+ } -+ return testbit128(sub, n % 128); -+} -+ -+/**** -+ **** 512-bit Primitives -+ ****/ -+ -+static really_inline m512 zeroes512(void) { -+ m512 rv = {zeroes256(), zeroes256()}; -+ return rv; -+} -+ -+static really_inline m512 ones512(void) { -+ m512 rv = {ones256(), ones256()}; -+ return rv; -+} -+ -+static really_inline m512 and512(m512 a, m512 b) { -+ m512 rv; -+ rv.lo = and256(a.lo, b.lo); -+ rv.hi = and256(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m512 or512(m512 a, m512 b) { -+ m512 rv; -+ rv.lo = or256(a.lo, b.lo); -+ rv.hi = or256(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m512 xor512(m512 a, m512 b) { -+ m512 rv; -+ rv.lo = xor256(a.lo, b.lo); -+ rv.hi = xor256(a.hi, b.hi); -+ return rv; -+} -+ -+static really_inline m512 not512(m512 a) { -+ m512 rv; -+ rv.lo = not256(a.lo); -+ rv.hi = not256(a.hi); -+ return rv; -+} -+ -+static really_inline m512 andnot512(m512 a, m512 b) { -+ m512 rv; -+ rv.lo = andnot256(a.lo, b.lo); -+ rv.hi = andnot256(a.hi, b.hi); -+ return rv; -+} -+ -+static really_really_inline m512 lshift64_m512(m512 a, unsigned b) { -+ m512 rv; -+ rv.lo = lshift64_m256(a.lo, b); -+ rv.hi = lshift64_m256(a.hi, b); -+ return rv; -+} -+ -+static really_inline int diff512(m512 a, m512 b) { -+ return diff256(a.lo, b.lo) || diff256(a.hi, b.hi); -+} -+ -+static really_inline int isnonzero512(m512 a) { -+ m128 x = or128(a.lo.lo, a.lo.hi); -+ m128 y = or128(a.hi.lo, a.hi.hi); -+ return isnonzero128(or128(x, y)); -+} -+ -+/** -+ * "Rich" version of diff512(). Takes two vectors a and b and returns a 16-bit -+ * mask indicating which 32-bit words contain differences. -+ */ -+static really_inline u32 diffrich512(m512 a, m512 b) { -+ uint32x4_t x = vceqq_s32(a.lo.lo.vect_s32, b.lo.lo.vect_s32); -+ uint32x4_t y = vceqq_s32(a.lo.hi.vect_s32, b.lo.hi.vect_s32); -+ uint32x4_t z = vceqq_s32(a.hi.lo.vect_s32, b.hi.lo.vect_s32); -+ uint32x4_t w = vceqq_s32(a.hi.hi.vect_s32, b.hi.hi.vect_s32); -+ uint16x8_t p = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y)); -+ uint16x8_t q = vcombine_u16(vqmovn_u32(z), vqmovn_u32(w)); -+ -+ uint8x16_t input = vcombine_u8(vqmovn_u16(p), vqmovn_u16(q)); -+ -+ static const int8_t __attribute__((aligned(16))) -+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; -+ uint8x8_t mask_and = vdup_n_u8(0x80); -+ int8x8_t mask_shift = vld1_s8(xr); -+ -+ uint8x8_t lo = vget_low_u8(input); -+ uint8x8_t hi = vget_high_u8(input); -+ -+ lo = vand_u8(lo, mask_and); -+ lo = vshl_u8(lo, mask_shift); -+ -+ hi = vand_u8(hi, mask_and); -+ hi = vshl_u8(hi, mask_shift); -+ -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ lo = vpadd_u8(lo, lo); -+ -+ hi = vpadd_u8(hi, hi); -+ hi = vpadd_u8(hi, hi); -+ hi = vpadd_u8(hi, hi); -+ -+ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xffff; -+} -+ -+/** -+ * "Rich" version of diffrich(), 64-bit variant. Takes two vectors a and b and -+ * returns a 16-bit mask indicating which 64-bit words contain differences. -+ */ -+static really_inline u32 diffrich64_512(m512 a, m512 b) { -+ u32 d = diffrich512(a, b); -+ return (d | (d >> 1)) & 0x55555555; -+} -+ -+// aligned load -+static really_inline m512 load512(const void *ptr) { -+ assert(ISALIGNED_N(ptr, alignof(m256))); -+ m512 rv = {load256(ptr), load256((const char *)ptr + 32)}; -+ return rv; -+} -+ -+// aligned store -+static really_inline void store512(void *ptr, m512 a) { -+ assert(ISALIGNED_N(ptr, alignof(m512))); -+ ptr = assume_aligned(ptr, 16); -+ *(m512 *)ptr = a; -+} -+ -+// unaligned load -+static really_inline m512 loadu512(const void *ptr) { -+ m512 rv = {loadu256(ptr), loadu256((const char *)ptr + 32)}; -+ return rv; -+} -+ -+// packed unaligned store of first N bytes -+static really_inline void storebytes512(void *ptr, m512 a, unsigned int n) { -+ assert(n <= sizeof(a)); -+ memcpy(ptr, &a, n); -+} -+ -+// packed unaligned load of first N bytes, pad with zero -+static really_inline m512 loadbytes512(const void *ptr, unsigned int n) { -+ m512 a = zeroes512(); -+ assert(n <= sizeof(a)); -+ memcpy(&a, ptr, n); -+ return a; -+} -+ -+static really_inline m512 mask1bit512(unsigned int n) { -+ assert(n < sizeof(m512) * 8); -+ u32 mask_idx = ((n % 8) * 64) + 95; -+ mask_idx -= n / 8; -+ return loadu512(&simd_onebit_masks[mask_idx]); -+} -+ -+// switches on bit N in the given vector. -+static really_inline void setbit512(m512 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo.lo; -+ } else if (n < 256) { -+ sub = &ptr->lo.hi; -+ } else if (n < 384) { -+ sub = &ptr->hi.lo; -+ } else { -+ sub = &ptr->hi.hi; -+ } -+ setbit128(sub, n % 128); -+} -+ -+// switches off bit N in the given vector. -+static really_inline void clearbit512(m512 *ptr, unsigned int n) { -+ assert(n < sizeof(*ptr) * 8); -+ m128 *sub; -+ if (n < 128) { -+ sub = &ptr->lo.lo; -+ } else if (n < 256) { -+ sub = &ptr->lo.hi; -+ } else if (n < 384) { -+ sub = &ptr->hi.lo; -+ } else { -+ sub = &ptr->hi.hi; -+ } -+ clearbit128(sub, n % 128); -+} -+ -+// tests bit N in the given vector. -+static really_inline char testbit512(m512 val, unsigned int n) { -+ assert(n < sizeof(val) * 8); -+ m128 sub; -+ if (n < 128) { -+ sub = val.lo.lo; -+ } else if (n < 256) { -+ sub = val.lo.hi; -+ } else if (n < 384) { -+ sub = val.hi.lo; -+ } else { -+ sub = val.hi.hi; -+ } -+ return testbit128(sub, n % 128); -+} -+#pragma GCC diagnostic pop + /* + * ICC and MSVC don't break out POPCNT or BMI/2 as separate pre-def macros + */ +@@ -87,4 +91,11 @@ + #define NO_ASM + #endif + ++/* ++ * AARCH64 uses a different form of inline asm ++ */ ++#if defined(__aarch64__) ++#define NO_ASM ++#endif + + #endif // UTIL_ARCH_H_ +diff --git a/src/util/cpuid_flags.c b/src/util/cpuid_flags.c +index c00ce58..96286ee 100644 +--- a/src/util/cpuid_flags.c ++++ b/src/util/cpuid_flags.c +@@ -40,6 +40,7 @@ + u64a cpuid_flags(void) { + u64a cap = 0; + ++#if defined(__X86_64__) + if (check_avx2()) { + DEBUG_PRINTF("AVX2 enabled\n"); + cap |= HS_CPU_FEATURES_AVX2; +@@ -67,6 +68,7 @@ u64a cpuid_flags(void) { + #if (!defined(FAT_RUNTIME) && !defined(HAVE_AVX512VBMI)) || \ + (defined(FAT_RUNTIME) && !defined(BUILD_AVX512VBMI)) + cap &= ~HS_CPU_FEATURES_AVX512VBMI; +#endif -diff --git a/src/util/simd_types.h b/src/util/simd_types.h -index 962cad6..62d39ec 100644 ---- a/src/util/simd_types.h -+++ b/src/util/simd_types.h -@@ -30,28 +30,58 @@ - #define SIMD_TYPES_H + #endif - #include "config.h" -+#include "ue2common.h" - #include "util/arch.h" - #include "util/intrinsics.h" --#include "ue2common.h" + return cap; +@@ -78,6 +80,7 @@ struct family_id { + u32 tune; + }; - #if defined(HAVE_SSE2) -+typedef __m128i m128; -+#elif defined(HAVE_NEON) -+#include "arm_neon.h" -+ -+typedef union { -+ int8x16_t vect_s8; -+ int16x8_t vect_s16; -+ int32x4_t vect_s32; -+ int64x2_t vect_s64; -+ uint8x16_t vect_u8; -+ uint16x8_t vect_u16; -+ uint32x4_t vect_u32; -+ uint64x2_t vect_u64; -+} __m128i; -+typedef float32x4_t __m128; -+typedef float64x2_t __m128d; -+ - typedef __m128i m128; - #else --typedef struct ALIGN_DIRECTIVE {u64a hi; u64a lo;} m128; -+typedef struct ALIGN_DIRECTIVE { -+ u64a hi; -+ u64a lo; -+} m128; -+ - #endif ++#if defined(__X86_64__) + /* from table 35-1 of the Intel 64 and IA32 Arch. Software Developer's Manual + * and "Intel Architecture and Processor Identification With CPUID Model and + * Family Numbers" */ +@@ -121,6 +124,7 @@ static const struct family_id known_microarch[] = { + { 0x6, 0x6C, HS_TUNE_FAMILY_ICX }, /* Icelake Xeon */ - #if defined(HAVE_AVX2) - typedef __m256i m256; - #else --typedef struct ALIGN_AVX_DIRECTIVE {m128 lo; m128 hi;} m256; -+typedef struct ALIGN_AVX_DIRECTIVE { -+ m128 lo; -+ m128 hi; -+} m256; - #endif + }; ++#endif --typedef struct {m128 lo; m128 mid; m128 hi;} m384; -+typedef struct { -+ m128 lo; -+ m128 mid; -+ m128 hi; -+} m384; - #if defined(HAVE_AVX512) - typedef __m512i m512; - #else --typedef struct ALIGN_ATTR(64) {m256 lo; m256 hi;} m512; -+typedef struct ALIGN_ATTR(64) { -+ m256 lo; -+ m256 hi; -+} m512; + #ifdef DUMP_SUPPORT + static UNUSED +@@ -144,6 +148,7 @@ const char *dumpTune(u32 tune) { #endif - #endif /* SIMD_TYPES_H */ -- -diff --git a/src/util/simd_utils.h b/src/util/simd_utils.h -index d1f060b..7e926b2 100644 ---- a/src/util/simd_utils.h -+++ b/src/util/simd_utils.h -@@ -26,1395 +26,14 @@ - * POSSIBILITY OF SUCH DAMAGE. - */ - --/** \file -- * \brief SIMD types and primitive operations. -- */ -- - #ifndef SIMD_UTILS - #define SIMD_UTILS + u32 cpuid_tune(void) { ++#if defined(__X86_64__) + unsigned int eax, ebx, ecx, edx; --#if !defined(_WIN32) && !defined(__SSSE3__) --#error SSSE3 instructions must be enabled --#endif -- --#include "config.h" --#include "ue2common.h" --#include "simd_types.h" --#include "unaligned.h" --#include "util/arch.h" --#include "util/intrinsics.h" -- --#include // for memcpy -- --// Define a common assume_aligned using an appropriate compiler built-in, if --// it's available. Note that we need to handle C or C++ compilation. --#ifdef __cplusplus --# ifdef HAVE_CXX_BUILTIN_ASSUME_ALIGNED --# define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) --# endif --#else --# ifdef HAVE_CC_BUILTIN_ASSUME_ALIGNED --# define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) --# endif --#endif -- --// Fallback to identity case. --#ifndef assume_aligned --#define assume_aligned(x, y) (x) --#endif -- --#ifdef __cplusplus --extern "C" { --#endif --extern const char vbs_mask_data[]; --#ifdef __cplusplus --} --#endif -- --static really_inline m128 ones128(void) { --#if defined(__GNUC__) || defined(__INTEL_COMPILER) -- /* gcc gets this right */ -- return _mm_set1_epi8(0xFF); --#else -- /* trick from Intel's optimization guide to generate all-ones. -- * ICC converts this to the single cmpeq instruction */ -- return _mm_cmpeq_epi8(_mm_setzero_si128(), _mm_setzero_si128()); --#endif --} -- --static really_inline m128 zeroes128(void) { -- return _mm_setzero_si128(); --} -- --/** \brief Bitwise not for m128*/ --static really_inline m128 not128(m128 a) { -- return _mm_xor_si128(a, ones128()); --} -- --/** \brief Return 1 if a and b are different otherwise 0 */ --static really_inline int diff128(m128 a, m128 b) { -- return (_mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) ^ 0xffff); --} -- --static really_inline int isnonzero128(m128 a) { -- return !!diff128(a, zeroes128()); --} -- --/** -- * "Rich" version of diff128(). Takes two vectors a and b and returns a 4-bit -- * mask indicating which 32-bit words contain differences. -- */ --static really_inline u32 diffrich128(m128 a, m128 b) { -- a = _mm_cmpeq_epi32(a, b); -- return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0xf; --} -- --/** -- * "Rich" version of diff128(), 64-bit variant. Takes two vectors a and b and -- * returns a 4-bit mask indicating which 64-bit words contain differences. -- */ --static really_inline u32 diffrich64_128(m128 a, m128 b) { --#if defined(HAVE_SSE41) -- a = _mm_cmpeq_epi64(a, b); -- return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0x5; --#else -- u32 d = diffrich128(a, b); -- return (d | (d >> 1)) & 0x5; --#endif --} -- --static really_really_inline --m128 lshift64_m128(m128 a, unsigned b) { --#if defined(HAVE__BUILTIN_CONSTANT_P) -- if (__builtin_constant_p(b)) { -- return _mm_slli_epi64(a, b); -- } --#endif -- m128 x = _mm_cvtsi32_si128(b); -- return _mm_sll_epi64(a, x); --} -- --#define rshift64_m128(a, b) _mm_srli_epi64((a), (b)) --#define eq128(a, b) _mm_cmpeq_epi8((a), (b)) --#define movemask128(a) ((u32)_mm_movemask_epi8((a))) -- --#if defined(HAVE_AVX512) --static really_inline m128 cast512to128(const m512 in) { -- return _mm512_castsi512_si128(in); --} --#endif -- --static really_inline m128 set16x8(u8 c) { -- return _mm_set1_epi8(c); --} -- --static really_inline m128 set4x32(u32 c) { -- return _mm_set1_epi32(c); --} -- --static really_inline u32 movd(const m128 in) { -- return _mm_cvtsi128_si32(in); --} -- --#if defined(HAVE_AVX512) --static really_inline u32 movd512(const m512 in) { -- // NOTE: seems gcc doesn't support _mm512_cvtsi512_si32(in), -- // so we use 2-step convertions to work around. -- return _mm_cvtsi128_si32(_mm512_castsi512_si128(in)); --} -- --static really_inline u64a movq512(const m512 in) { -- // NOTE: seems AVX512 doesn't support _mm512_cvtsi512_si64(in), -- // so we use 2-step convertions to work around. -- return _mm_cvtsi128_si64(_mm512_castsi512_si128(in)); --} --#endif -- --static really_inline u64a movq(const m128 in) { --#if defined(ARCH_X86_64) -- return _mm_cvtsi128_si64(in); --#else // 32-bit - this is horrific -- u32 lo = movd(in); -- u32 hi = movd(_mm_srli_epi64(in, 32)); -- return (u64a)hi << 32 | lo; --#endif --} -- --/* another form of movq */ --static really_inline --m128 load_m128_from_u64a(const u64a *p) { -- return _mm_set_epi64x(0LL, *p); --} -- --#define rshiftbyte_m128(a, count_immed) _mm_srli_si128(a, count_immed) --#define lshiftbyte_m128(a, count_immed) _mm_slli_si128(a, count_immed) -- --#if defined(HAVE_SSE41) --#define extract32from128(a, imm) _mm_extract_epi32(a, imm) --#define extract64from128(a, imm) _mm_extract_epi64(a, imm) --#else --#define extract32from128(a, imm) movd(_mm_srli_si128(a, imm << 2)) --#define extract64from128(a, imm) movq(_mm_srli_si128(a, imm << 3)) --#endif -- --#if !defined(HAVE_AVX2) --// TODO: this entire file needs restructuring - this carveout is awful --#define extractlow64from256(a) movq(a.lo) --#define extractlow32from256(a) movd(a.lo) --#if defined(HAVE_SSE41) --#define extract32from256(a, imm) _mm_extract_epi32((imm >> 2) ? a.hi : a.lo, imm % 4) --#define extract64from256(a, imm) _mm_extract_epi64((imm >> 1) ? a.hi : a.lo, imm % 2) --#else --#define extract32from256(a, imm) movd(_mm_srli_si128((imm >> 2) ? a.hi : a.lo, (imm % 4) * 4)) --#define extract64from256(a, imm) movq(_mm_srli_si128((imm >> 1) ? a.hi : a.lo, (imm % 2) * 8)) --#endif -- --#endif // !AVX2 -- --static really_inline m128 and128(m128 a, m128 b) { -- return _mm_and_si128(a,b); --} -- --static really_inline m128 xor128(m128 a, m128 b) { -- return _mm_xor_si128(a,b); --} -- --static really_inline m128 or128(m128 a, m128 b) { -- return _mm_or_si128(a,b); --} -- --#if defined(HAVE_AVX512VBMI) --static really_inline m512 expand128(m128 a) { -- return _mm512_broadcast_i32x4(a); --} -- --static really_inline m512 expand256(m256 a) { -- return _mm512_broadcast_i64x4(a); --} -- --static really_inline m512 expand384(m384 a) { -- u64a *lo = (u64a*)&a.lo; -- u64a *mid = (u64a*)&a.mid; -- u64a *hi = (u64a*)&a.hi; -- return _mm512_set_epi64(0ULL, 0ULL, hi[1], hi[0], mid[1], mid[0], -- lo[1], lo[0]); --} -+#if defined(__x86_64__) -+#include "simd_x86.h" -+#elif defined(__aarch64__) -+#include "simd_arm.h" - #endif + cpuid(1, 0, &eax, &ebx, &ecx, &edx); +@@ -171,6 +176,7 @@ u32 cpuid_tune(void) { + DEBUG_PRINTF("found tune flag %s\n", dumpTune(tune) ); + return tune; + } ++#endif --static really_inline m128 andnot128(m128 a, m128 b) { -- return _mm_andnot_si128(a, b); --} -- --// aligned load --static really_inline m128 load128(const void *ptr) { -- assert(ISALIGNED_N(ptr, alignof(m128))); -- ptr = assume_aligned(ptr, 16); -- return _mm_load_si128((const m128 *)ptr); --} -- --// aligned store --static really_inline void store128(void *ptr, m128 a) { -- assert(ISALIGNED_N(ptr, alignof(m128))); -- ptr = assume_aligned(ptr, 16); -- *(m128 *)ptr = a; --} -- --// unaligned load --static really_inline m128 loadu128(const void *ptr) { -- return _mm_loadu_si128((const m128 *)ptr); --} -- --// unaligned store --static really_inline void storeu128(void *ptr, m128 a) { -- _mm_storeu_si128 ((m128 *)ptr, a); --} -- --// packed unaligned store of first N bytes --static really_inline --void storebytes128(void *ptr, m128 a, unsigned int n) { -- assert(n <= sizeof(a)); -- memcpy(ptr, &a, n); --} -- --// packed unaligned load of first N bytes, pad with zero --static really_inline --m128 loadbytes128(const void *ptr, unsigned int n) { -- m128 a = zeroes128(); -- assert(n <= sizeof(a)); -- memcpy(&a, ptr, n); -- return a; --} -- --#ifdef __cplusplus --extern "C" { --#endif --extern const u8 simd_onebit_masks[]; --#ifdef __cplusplus --} - #endif + return HS_TUNE_FAMILY_GENERIC; + } +diff --git a/src/util/cpuid_flags.h b/src/util/cpuid_flags.h +index 527c6d5..3125bd1 100644 +--- a/src/util/cpuid_flags.h ++++ b/src/util/cpuid_flags.h +@@ -32,7 +32,9 @@ + #include "ue2common.h" --static really_inline --m128 mask1bit128(unsigned int n) { -- assert(n < sizeof(m128) * 8); -- u32 mask_idx = ((n % 8) * 64) + 95; -- mask_idx -= n / 8; -- return loadu128(&simd_onebit_masks[mask_idx]); --} -- --// switches on bit N in the given vector. --static really_inline --void setbit128(m128 *ptr, unsigned int n) { -- *ptr = or128(mask1bit128(n), *ptr); --} -- --// switches off bit N in the given vector. --static really_inline --void clearbit128(m128 *ptr, unsigned int n) { -- *ptr = andnot128(mask1bit128(n), *ptr); --} -- --// tests bit N in the given vector. --static really_inline --char testbit128(m128 val, unsigned int n) { -- const m128 mask = mask1bit128(n); --#if defined(HAVE_SSE41) -- return !_mm_testz_si128(mask, val); --#else -- return isnonzero128(and128(mask, val)); --#endif --} -- --// offset must be an immediate --#define palignr(r, l, offset) _mm_alignr_epi8(r, l, offset) -- --static really_inline --m128 pshufb_m128(m128 a, m128 b) { -- m128 result; -- result = _mm_shuffle_epi8(a, b); -- return result; --} -- --static really_inline --m256 pshufb_m256(m256 a, m256 b) { --#if defined(HAVE_AVX2) -- return _mm256_shuffle_epi8(a, b); --#else -- m256 rv; -- rv.lo = pshufb_m128(a.lo, b.lo); -- rv.hi = pshufb_m128(a.hi, b.hi); -- return rv; --#endif --} -- --#if defined(HAVE_AVX512) --static really_inline --m512 pshufb_m512(m512 a, m512 b) { -- return _mm512_shuffle_epi8(a, b); --} -- --static really_inline --m512 maskz_pshufb_m512(__mmask64 k, m512 a, m512 b) { -- return _mm512_maskz_shuffle_epi8(k, a, b); --} -- --#if defined(HAVE_AVX512VBMI) --#define vpermb512(idx, a) _mm512_permutexvar_epi8(idx, a) --#define maskz_vpermb512(k, idx, a) _mm512_maskz_permutexvar_epi8(k, idx, a) --#endif -- --#endif -- --static really_inline --m128 variable_byte_shift_m128(m128 in, s32 amount) { -- assert(amount >= -16 && amount <= 16); -- m128 shift_mask = loadu128(vbs_mask_data + 16 - amount); -- return pshufb_m128(in, shift_mask); --} -- --static really_inline --m128 max_u8_m128(m128 a, m128 b) { -- return _mm_max_epu8(a, b); --} -- --static really_inline --m128 min_u8_m128(m128 a, m128 b) { -- return _mm_min_epu8(a, b); --} -- --static really_inline --m128 sadd_u8_m128(m128 a, m128 b) { -- return _mm_adds_epu8(a, b); --} -- --static really_inline --m128 sub_u8_m128(m128 a, m128 b) { -- return _mm_sub_epi8(a, b); --} -- --static really_inline --m128 set64x2(u64a hi, u64a lo) { -- return _mm_set_epi64x(hi, lo); --} -- --/**** -- **** 256-bit Primitives -- ****/ -- --#if defined(HAVE_AVX2) -- --static really_really_inline --m256 lshift64_m256(m256 a, unsigned b) { --#if defined(HAVE__BUILTIN_CONSTANT_P) -- if (__builtin_constant_p(b)) { -- return _mm256_slli_epi64(a, b); -- } --#endif -- m128 x = _mm_cvtsi32_si128(b); -- return _mm256_sll_epi64(a, x); --} -- --#define rshift64_m256(a, b) _mm256_srli_epi64((a), (b)) -- --static really_inline --m256 set32x8(u32 in) { -- return _mm256_set1_epi8(in); --} -- --#define eq256(a, b) _mm256_cmpeq_epi8((a), (b)) --#define movemask256(a) ((u32)_mm256_movemask_epi8((a))) -- --static really_inline --m256 set2x128(m128 a) { -- return _mm256_broadcastsi128_si256(a); --} -- --#else -- --static really_really_inline --m256 lshift64_m256(m256 a, int b) { -- m256 rv = a; -- rv.lo = lshift64_m128(rv.lo, b); -- rv.hi = lshift64_m128(rv.hi, b); -- return rv; --} -- --static really_inline --m256 rshift64_m256(m256 a, int b) { -- m256 rv = a; -- rv.lo = rshift64_m128(rv.lo, b); -- rv.hi = rshift64_m128(rv.hi, b); -- return rv; --} --static really_inline --m256 set32x8(u32 in) { -- m256 rv; -- rv.lo = set16x8((u8) in); -- rv.hi = rv.lo; -- return rv; --} -- --static really_inline --m256 eq256(m256 a, m256 b) { -- m256 rv; -- rv.lo = eq128(a.lo, b.lo); -- rv.hi = eq128(a.hi, b.hi); -- return rv; --} -- --static really_inline --u32 movemask256(m256 a) { -- u32 lo_mask = movemask128(a.lo); -- u32 hi_mask = movemask128(a.hi); -- return lo_mask | (hi_mask << 16); --} -- --static really_inline --m256 set2x128(m128 a) { -- m256 rv = {a, a}; -- return rv; --} --#endif -- --static really_inline m256 zeroes256(void) { --#if defined(HAVE_AVX2) -- return _mm256_setzero_si256(); --#else -- m256 rv = {zeroes128(), zeroes128()}; -- return rv; --#endif --} -- --static really_inline m256 ones256(void) { --#if defined(HAVE_AVX2) -- m256 rv = _mm256_set1_epi8(0xFF); --#else -- m256 rv = {ones128(), ones128()}; --#endif -- return rv; --} -- --#if defined(HAVE_AVX2) --static really_inline m256 and256(m256 a, m256 b) { -- return _mm256_and_si256(a, b); --} --#else --static really_inline m256 and256(m256 a, m256 b) { -- m256 rv; -- rv.lo = and128(a.lo, b.lo); -- rv.hi = and128(a.hi, b.hi); -- return rv; --} --#endif -- --#if defined(HAVE_AVX2) --static really_inline m256 or256(m256 a, m256 b) { -- return _mm256_or_si256(a, b); --} --#else --static really_inline m256 or256(m256 a, m256 b) { -- m256 rv; -- rv.lo = or128(a.lo, b.lo); -- rv.hi = or128(a.hi, b.hi); -- return rv; --} --#endif -- --#if defined(HAVE_AVX2) --static really_inline m256 xor256(m256 a, m256 b) { -- return _mm256_xor_si256(a, b); --} --#else --static really_inline m256 xor256(m256 a, m256 b) { -- m256 rv; -- rv.lo = xor128(a.lo, b.lo); -- rv.hi = xor128(a.hi, b.hi); -- return rv; --} --#endif -- --#if defined(HAVE_AVX2) --static really_inline m256 not256(m256 a) { -- return _mm256_xor_si256(a, ones256()); --} --#else --static really_inline m256 not256(m256 a) { -- m256 rv; -- rv.lo = not128(a.lo); -- rv.hi = not128(a.hi); -- return rv; --} --#endif -- --#if defined(HAVE_AVX2) --static really_inline m256 andnot256(m256 a, m256 b) { -- return _mm256_andnot_si256(a, b); --} --#else --static really_inline m256 andnot256(m256 a, m256 b) { -- m256 rv; -- rv.lo = andnot128(a.lo, b.lo); -- rv.hi = andnot128(a.hi, b.hi); -- return rv; --} --#endif -- --static really_inline int diff256(m256 a, m256 b) { --#if defined(HAVE_AVX2) -- return !!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(a, b)) ^ (int)-1); --#else -- return diff128(a.lo, b.lo) || diff128(a.hi, b.hi); --#endif --} -- --static really_inline int isnonzero256(m256 a) { --#if defined(HAVE_AVX2) -- return !!diff256(a, zeroes256()); --#else -- return isnonzero128(or128(a.lo, a.hi)); --#endif --} -- --/** -- * "Rich" version of diff256(). Takes two vectors a and b and returns an 8-bit -- * mask indicating which 32-bit words contain differences. -- */ --static really_inline u32 diffrich256(m256 a, m256 b) { --#if defined(HAVE_AVX2) -- a = _mm256_cmpeq_epi32(a, b); -- return ~(_mm256_movemask_ps(_mm256_castsi256_ps(a))) & 0xFF; --#else -- m128 z = zeroes128(); -- a.lo = _mm_cmpeq_epi32(a.lo, b.lo); -- a.hi = _mm_cmpeq_epi32(a.hi, b.hi); -- m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.hi), z); -- return ~(_mm_movemask_epi8(packed)) & 0xff; --#endif --} -- --/** -- * "Rich" version of diff256(), 64-bit variant. Takes two vectors a and b and -- * returns an 8-bit mask indicating which 64-bit words contain differences. -- */ --static really_inline u32 diffrich64_256(m256 a, m256 b) { -- u32 d = diffrich256(a, b); -- return (d | (d >> 1)) & 0x55555555; --} -- --// aligned load --static really_inline m256 load256(const void *ptr) { -- assert(ISALIGNED_N(ptr, alignof(m256))); --#if defined(HAVE_AVX2) -- return _mm256_load_si256((const m256 *)ptr); --#else -- m256 rv = { load128(ptr), load128((const char *)ptr + 16) }; -- return rv; --#endif --} -- --// aligned load of 128-bit value to low and high part of 256-bit value --static really_inline m256 load2x128(const void *ptr) { --#if defined(HAVE_AVX2) -- return set2x128(load128(ptr)); --#else -- assert(ISALIGNED_N(ptr, alignof(m128))); -- m256 rv; -- rv.hi = rv.lo = load128(ptr); -- return rv; --#endif --} -- --static really_inline m256 loadu2x128(const void *ptr) { -- return set2x128(loadu128(ptr)); --} -- --// aligned store --static really_inline void store256(void *ptr, m256 a) { -- assert(ISALIGNED_N(ptr, alignof(m256))); --#if defined(HAVE_AVX2) -- _mm256_store_si256((m256 *)ptr, a); --#else -- ptr = assume_aligned(ptr, 16); -- *(m256 *)ptr = a; --#endif --} -- --// unaligned load --static really_inline m256 loadu256(const void *ptr) { --#if defined(HAVE_AVX2) -- return _mm256_loadu_si256((const m256 *)ptr); --#else -- m256 rv = { loadu128(ptr), loadu128((const char *)ptr + 16) }; -- return rv; --#endif --} -- --// unaligned store --static really_inline void storeu256(void *ptr, m256 a) { --#if defined(HAVE_AVX2) -- _mm256_storeu_si256((m256 *)ptr, a); --#else -- storeu128(ptr, a.lo); -- storeu128((char *)ptr + 16, a.hi); --#endif --} -- --// packed unaligned store of first N bytes --static really_inline --void storebytes256(void *ptr, m256 a, unsigned int n) { -- assert(n <= sizeof(a)); -- memcpy(ptr, &a, n); --} -- --// packed unaligned load of first N bytes, pad with zero --static really_inline --m256 loadbytes256(const void *ptr, unsigned int n) { -- m256 a = zeroes256(); -- assert(n <= sizeof(a)); -- memcpy(&a, ptr, n); -- return a; --} -- --static really_inline --m256 mask1bit256(unsigned int n) { -- assert(n < sizeof(m256) * 8); -- u32 mask_idx = ((n % 8) * 64) + 95; -- mask_idx -= n / 8; -- return loadu256(&simd_onebit_masks[mask_idx]); --} -- --static really_inline --m256 set64x4(u64a hi_1, u64a hi_0, u64a lo_1, u64a lo_0) { --#if defined(HAVE_AVX2) -- return _mm256_set_epi64x(hi_1, hi_0, lo_1, lo_0); --#else -- m256 rv; -- rv.hi = set64x2(hi_1, hi_0); -- rv.lo = set64x2(lo_1, lo_0); -- return rv; --#endif --} -- --#if !defined(HAVE_AVX2) --// switches on bit N in the given vector. --static really_inline --void setbit256(m256 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo; -- } else { -- sub = &ptr->hi; -- n -= 128; -- } -- setbit128(sub, n); --} -- --// switches off bit N in the given vector. --static really_inline --void clearbit256(m256 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo; -- } else { -- sub = &ptr->hi; -- n -= 128; -- } -- clearbit128(sub, n); --} -- --// tests bit N in the given vector. --static really_inline --char testbit256(m256 val, unsigned int n) { -- assert(n < sizeof(val) * 8); -- m128 sub; -- if (n < 128) { -- sub = val.lo; -- } else { -- sub = val.hi; -- n -= 128; -- } -- return testbit128(sub, n); --} -- --static really_really_inline --m128 movdq_hi(m256 x) { -- return x.hi; --} -- --static really_really_inline --m128 movdq_lo(m256 x) { -- return x.lo; --} -- --static really_inline --m256 combine2x128(m128 hi, m128 lo) { -- m256 rv = {lo, hi}; -- return rv; --} -- --#else // AVX2 -- --// switches on bit N in the given vector. --static really_inline --void setbit256(m256 *ptr, unsigned int n) { -- *ptr = or256(mask1bit256(n), *ptr); --} -- --static really_inline --void clearbit256(m256 *ptr, unsigned int n) { -- *ptr = andnot256(mask1bit256(n), *ptr); --} -- --// tests bit N in the given vector. --static really_inline --char testbit256(m256 val, unsigned int n) { -- const m256 mask = mask1bit256(n); -- return !_mm256_testz_si256(mask, val); --} -- --static really_really_inline --m128 movdq_hi(m256 x) { -- return _mm256_extracti128_si256(x, 1); --} -- --static really_really_inline --m128 movdq_lo(m256 x) { -- return _mm256_extracti128_si256(x, 0); --} -- --#define cast256to128(a) _mm256_castsi256_si128(a) --#define cast128to256(a) _mm256_castsi128_si256(a) --#define swap128in256(a) _mm256_permute4x64_epi64(a, 0x4E) --#define insert128to256(a, b, imm) _mm256_inserti128_si256(a, b, imm) --#define rshift128_m256(a, count_immed) _mm256_srli_si256(a, count_immed) --#define lshift128_m256(a, count_immed) _mm256_slli_si256(a, count_immed) --#define extract64from256(a, imm) _mm_extract_epi64(_mm256_extracti128_si256(a, imm >> 1), imm % 2) --#define extract32from256(a, imm) _mm_extract_epi32(_mm256_extracti128_si256(a, imm >> 2), imm % 4) --#define extractlow64from256(a) _mm_cvtsi128_si64(cast256to128(a)) --#define extractlow32from256(a) movd(cast256to128(a)) --#define interleave256hi(a, b) _mm256_unpackhi_epi8(a, b) --#define interleave256lo(a, b) _mm256_unpacklo_epi8(a, b) --#define vpalignr(r, l, offset) _mm256_alignr_epi8(r, l, offset) -- --static really_inline --m256 combine2x128(m128 hi, m128 lo) { --#if defined(_mm256_set_m128i) -- return _mm256_set_m128i(hi, lo); --#else -- return insert128to256(cast128to256(lo), hi, 1); --#endif --} --#endif //AVX2 -- --#if defined(HAVE_AVX512) --#define extract128from512(a, imm) _mm512_extracti32x4_epi32(a, imm) --#define interleave512hi(a, b) _mm512_unpackhi_epi8(a, b) --#define interleave512lo(a, b) _mm512_unpacklo_epi8(a, b) --#define set2x256(a) _mm512_broadcast_i64x4(a) --#define mask_set2x256(src, k, a) _mm512_mask_broadcast_i64x4(src, k, a) --#define vpermq512(idx, a) _mm512_permutexvar_epi64(idx, a) --#endif -- --/**** -- **** 384-bit Primitives -- ****/ -- --static really_inline m384 and384(m384 a, m384 b) { -- m384 rv; -- rv.lo = and128(a.lo, b.lo); -- rv.mid = and128(a.mid, b.mid); -- rv.hi = and128(a.hi, b.hi); -- return rv; --} -- --static really_inline m384 or384(m384 a, m384 b) { -- m384 rv; -- rv.lo = or128(a.lo, b.lo); -- rv.mid = or128(a.mid, b.mid); -- rv.hi = or128(a.hi, b.hi); -- return rv; --} -- --static really_inline m384 xor384(m384 a, m384 b) { -- m384 rv; -- rv.lo = xor128(a.lo, b.lo); -- rv.mid = xor128(a.mid, b.mid); -- rv.hi = xor128(a.hi, b.hi); -- return rv; --} --static really_inline m384 not384(m384 a) { -- m384 rv; -- rv.lo = not128(a.lo); -- rv.mid = not128(a.mid); -- rv.hi = not128(a.hi); -- return rv; --} --static really_inline m384 andnot384(m384 a, m384 b) { -- m384 rv; -- rv.lo = andnot128(a.lo, b.lo); -- rv.mid = andnot128(a.mid, b.mid); -- rv.hi = andnot128(a.hi, b.hi); -- return rv; --} -- --static really_really_inline --m384 lshift64_m384(m384 a, unsigned b) { -- m384 rv; -- rv.lo = lshift64_m128(a.lo, b); -- rv.mid = lshift64_m128(a.mid, b); -- rv.hi = lshift64_m128(a.hi, b); -- return rv; --} -- --static really_inline m384 zeroes384(void) { -- m384 rv = {zeroes128(), zeroes128(), zeroes128()}; -- return rv; --} -- --static really_inline m384 ones384(void) { -- m384 rv = {ones128(), ones128(), ones128()}; -- return rv; --} -- --static really_inline int diff384(m384 a, m384 b) { -- return diff128(a.lo, b.lo) || diff128(a.mid, b.mid) || diff128(a.hi, b.hi); --} -- --static really_inline int isnonzero384(m384 a) { -- return isnonzero128(or128(or128(a.lo, a.mid), a.hi)); --} -- --/** -- * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit -- * mask indicating which 32-bit words contain differences. -- */ --static really_inline u32 diffrich384(m384 a, m384 b) { -- m128 z = zeroes128(); -- a.lo = _mm_cmpeq_epi32(a.lo, b.lo); -- a.mid = _mm_cmpeq_epi32(a.mid, b.mid); -- a.hi = _mm_cmpeq_epi32(a.hi, b.hi); -- m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.mid), -- _mm_packs_epi32(a.hi, z)); -- return ~(_mm_movemask_epi8(packed)) & 0xfff; --} -- --/** -- * "Rich" version of diff384(), 64-bit variant. Takes two vectors a and b and -- * returns a 12-bit mask indicating which 64-bit words contain differences. -- */ --static really_inline u32 diffrich64_384(m384 a, m384 b) { -- u32 d = diffrich384(a, b); -- return (d | (d >> 1)) & 0x55555555; --} -- --// aligned load --static really_inline m384 load384(const void *ptr) { -- assert(ISALIGNED_16(ptr)); -- m384 rv = { load128(ptr), load128((const char *)ptr + 16), -- load128((const char *)ptr + 32) }; -- return rv; --} -- --// aligned store --static really_inline void store384(void *ptr, m384 a) { -- assert(ISALIGNED_16(ptr)); -- ptr = assume_aligned(ptr, 16); -- *(m384 *)ptr = a; --} -- --// unaligned load --static really_inline m384 loadu384(const void *ptr) { -- m384 rv = { loadu128(ptr), loadu128((const char *)ptr + 16), -- loadu128((const char *)ptr + 32)}; -- return rv; --} -- --// packed unaligned store of first N bytes --static really_inline --void storebytes384(void *ptr, m384 a, unsigned int n) { -- assert(n <= sizeof(a)); -- memcpy(ptr, &a, n); --} -- --// packed unaligned load of first N bytes, pad with zero --static really_inline --m384 loadbytes384(const void *ptr, unsigned int n) { -- m384 a = zeroes384(); -- assert(n <= sizeof(a)); -- memcpy(&a, ptr, n); -- return a; --} -- --// switches on bit N in the given vector. --static really_inline --void setbit384(m384 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo; -- } else if (n < 256) { -- sub = &ptr->mid; -- } else { -- sub = &ptr->hi; -- } -- setbit128(sub, n % 128); --} -- --// switches off bit N in the given vector. --static really_inline --void clearbit384(m384 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo; -- } else if (n < 256) { -- sub = &ptr->mid; -- } else { -- sub = &ptr->hi; -- } -- clearbit128(sub, n % 128); --} -- --// tests bit N in the given vector. --static really_inline --char testbit384(m384 val, unsigned int n) { -- assert(n < sizeof(val) * 8); -- m128 sub; -- if (n < 128) { -- sub = val.lo; -- } else if (n < 256) { -- sub = val.mid; -- } else { -- sub = val.hi; -- } -- return testbit128(sub, n % 128); --} -- --/**** -- **** 512-bit Primitives -- ****/ -- --#define eq512mask(a, b) _mm512_cmpeq_epi8_mask((a), (b)) --#define masked_eq512mask(k, a, b) _mm512_mask_cmpeq_epi8_mask((k), (a), (b)) -- --static really_inline --m512 zeroes512(void) { --#if defined(HAVE_AVX512) -- return _mm512_setzero_si512(); --#else -- m512 rv = {zeroes256(), zeroes256()}; -- return rv; --#endif --} -- --static really_inline --m512 ones512(void) { --#if defined(HAVE_AVX512) -- return _mm512_set1_epi8(0xFF); -- //return _mm512_xor_si512(_mm512_setzero_si512(), _mm512_setzero_si512()); --#else -- m512 rv = {ones256(), ones256()}; -- return rv; --#endif --} -- --#if defined(HAVE_AVX512) --static really_inline --m512 set64x8(u8 a) { -- return _mm512_set1_epi8(a); --} -- --static really_inline --m512 set8x64(u64a a) { -- return _mm512_set1_epi64(a); --} -- --static really_inline --m512 set16x32(u32 a) { -- return _mm512_set1_epi32(a); --} -- --static really_inline --m512 set512_64(u64a hi_3, u64a hi_2, u64a hi_1, u64a hi_0, -- u64a lo_3, u64a lo_2, u64a lo_1, u64a lo_0) { -- return _mm512_set_epi64(hi_3, hi_2, hi_1, hi_0, -- lo_3, lo_2, lo_1, lo_0); --} -- --static really_inline --m512 swap256in512(m512 a) { -- m512 idx = set512_64(3ULL, 2ULL, 1ULL, 0ULL, 7ULL, 6ULL, 5ULL, 4ULL); -- return vpermq512(idx, a); --} -- --static really_inline --m512 set4x128(m128 a) { -- return _mm512_broadcast_i32x4(a); --} -- --static really_inline --m512 sadd_u8_m512(m512 a, m512 b) { -- return _mm512_adds_epu8(a, b); --} -- --static really_inline --m512 max_u8_m512(m512 a, m512 b) { -- return _mm512_max_epu8(a, b); --} -- --static really_inline --m512 min_u8_m512(m512 a, m512 b) { -- return _mm512_min_epu8(a, b); --} -- --static really_inline --m512 sub_u8_m512(m512 a, m512 b) { -- return _mm512_sub_epi8(a, b); --} --#endif -- --static really_inline --m512 and512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return _mm512_and_si512(a, b); --#else -- m512 rv; -- rv.lo = and256(a.lo, b.lo); -- rv.hi = and256(a.hi, b.hi); -- return rv; --#endif --} -- --static really_inline --m512 or512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return _mm512_or_si512(a, b); --#else -- m512 rv; -- rv.lo = or256(a.lo, b.lo); -- rv.hi = or256(a.hi, b.hi); -- return rv; --#endif --} -- --static really_inline --m512 xor512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return _mm512_xor_si512(a, b); --#else -- m512 rv; -- rv.lo = xor256(a.lo, b.lo); -- rv.hi = xor256(a.hi, b.hi); -- return rv; --#endif --} -- --static really_inline --m512 not512(m512 a) { --#if defined(HAVE_AVX512) -- return _mm512_xor_si512(a, ones512()); --#else -- m512 rv; -- rv.lo = not256(a.lo); -- rv.hi = not256(a.hi); -- return rv; --#endif --} -- --static really_inline --m512 andnot512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return _mm512_andnot_si512(a, b); --#else -- m512 rv; -- rv.lo = andnot256(a.lo, b.lo); -- rv.hi = andnot256(a.hi, b.hi); -- return rv; --#endif --} -- --#if defined(HAVE_AVX512) --static really_really_inline --m512 lshift64_m512(m512 a, unsigned b) { --#if defined(HAVE__BUILTIN_CONSTANT_P) -- if (__builtin_constant_p(b)) { -- return _mm512_slli_epi64(a, b); -- } --#endif -- m128 x = _mm_cvtsi32_si128(b); -- return _mm512_sll_epi64(a, x); --} --#else --static really_really_inline --m512 lshift64_m512(m512 a, unsigned b) { -- m512 rv; -- rv.lo = lshift64_m256(a.lo, b); -- rv.hi = lshift64_m256(a.hi, b); -- return rv; --} --#endif -- --#if defined(HAVE_AVX512) --#define rshift64_m512(a, b) _mm512_srli_epi64((a), (b)) --#define rshift128_m512(a, count_immed) _mm512_bsrli_epi128(a, count_immed) --#define lshift128_m512(a, count_immed) _mm512_bslli_epi128(a, count_immed) --#endif -- --#if !defined(_MM_CMPINT_NE) --#define _MM_CMPINT_NE 0x4 --#endif -- --static really_inline --int diff512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return !!_mm512_cmp_epi8_mask(a, b, _MM_CMPINT_NE); --#else -- return diff256(a.lo, b.lo) || diff256(a.hi, b.hi); --#endif --} -- --static really_inline --int isnonzero512(m512 a) { --#if defined(HAVE_AVX512) -- return diff512(a, zeroes512()); --#elif defined(HAVE_AVX2) -- m256 x = or256(a.lo, a.hi); -- return !!diff256(x, zeroes256()); --#else -- m128 x = or128(a.lo.lo, a.lo.hi); -- m128 y = or128(a.hi.lo, a.hi.hi); -- return isnonzero128(or128(x, y)); --#endif --} -- --/** -- * "Rich" version of diff512(). Takes two vectors a and b and returns a 16-bit -- * mask indicating which 32-bit words contain differences. -- */ --static really_inline --u32 diffrich512(m512 a, m512 b) { --#if defined(HAVE_AVX512) -- return _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_NE); --#elif defined(HAVE_AVX2) -- return diffrich256(a.lo, b.lo) | (diffrich256(a.hi, b.hi) << 8); --#else -- a.lo.lo = _mm_cmpeq_epi32(a.lo.lo, b.lo.lo); -- a.lo.hi = _mm_cmpeq_epi32(a.lo.hi, b.lo.hi); -- a.hi.lo = _mm_cmpeq_epi32(a.hi.lo, b.hi.lo); -- a.hi.hi = _mm_cmpeq_epi32(a.hi.hi, b.hi.hi); -- m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo.lo, a.lo.hi), -- _mm_packs_epi32(a.hi.lo, a.hi.hi)); -- return ~(_mm_movemask_epi8(packed)) & 0xffff; --#endif --} -- --/** -- * "Rich" version of diffrich(), 64-bit variant. Takes two vectors a and b and -- * returns a 16-bit mask indicating which 64-bit words contain differences. -- */ --static really_inline --u32 diffrich64_512(m512 a, m512 b) { -- //TODO: cmp_epi64? -- u32 d = diffrich512(a, b); -- return (d | (d >> 1)) & 0x55555555; --} -- --// aligned load --static really_inline --m512 load512(const void *ptr) { --#if defined(HAVE_AVX512) -- return _mm512_load_si512(ptr); --#else -- assert(ISALIGNED_N(ptr, alignof(m256))); -- m512 rv = { load256(ptr), load256((const char *)ptr + 32) }; -- return rv; --#endif --} -- --// aligned store --static really_inline --void store512(void *ptr, m512 a) { -- assert(ISALIGNED_N(ptr, alignof(m512))); --#if defined(HAVE_AVX512) -- return _mm512_store_si512(ptr, a); --#elif defined(HAVE_AVX2) -- m512 *x = (m512 *)ptr; -- store256(&x->lo, a.lo); -- store256(&x->hi, a.hi); --#else -- ptr = assume_aligned(ptr, 16); -- *(m512 *)ptr = a; --#endif --} -- --// unaligned load --static really_inline --m512 loadu512(const void *ptr) { --#if defined(HAVE_AVX512) -- return _mm512_loadu_si512(ptr); --#else -- m512 rv = { loadu256(ptr), loadu256((const char *)ptr + 32) }; -- return rv; --#endif --} -- --// unaligned store --static really_inline --void storeu512(void *ptr, m512 a) { --#if defined(HAVE_AVX512) -- _mm512_storeu_si512((m512 *)ptr, a); --#elif defined(HAVE_AVX2) -- storeu256(ptr, a.lo); -- storeu256((char *)ptr + 32, a.hi); --#else -- storeu128(ptr, a.lo.lo); -- storeu128((char *)ptr + 16, a.lo.hi); -- storeu128((char *)ptr + 32, a.hi.lo); -- storeu128((char *)ptr + 48, a.hi.hi); --#endif --} -- --#if defined(HAVE_AVX512) --static really_inline --m512 loadu_maskz_m512(__mmask64 k, const void *ptr) { -- return _mm512_maskz_loadu_epi8(k, ptr); --} -- --static really_inline --m512 loadu_mask_m512(m512 src, __mmask64 k, const void *ptr) { -- return _mm512_mask_loadu_epi8(src, k, ptr); --} -- --static really_inline --void storeu_mask_m512(void *ptr, __mmask64 k, m512 a) { -- _mm512_mask_storeu_epi8(ptr, k, a); --} -- --static really_inline --m512 set_mask_m512(__mmask64 k) { -- return _mm512_movm_epi8(k); --} -- --static really_inline --m256 loadu_maskz_m256(__mmask32 k, const void *ptr) { -- return _mm256_maskz_loadu_epi8(k, ptr); --} --#endif -- --// packed unaligned store of first N bytes --static really_inline --void storebytes512(void *ptr, m512 a, unsigned int n) { -- assert(n <= sizeof(a)); -- memcpy(ptr, &a, n); --} -- --// packed unaligned load of first N bytes, pad with zero --static really_inline --m512 loadbytes512(const void *ptr, unsigned int n) { -- m512 a = zeroes512(); -- assert(n <= sizeof(a)); -- memcpy(&a, ptr, n); -- return a; --} -- --static really_inline --m512 mask1bit512(unsigned int n) { -- assert(n < sizeof(m512) * 8); -- u32 mask_idx = ((n % 8) * 64) + 95; -- mask_idx -= n / 8; -- return loadu512(&simd_onebit_masks[mask_idx]); --} -- --// switches on bit N in the given vector. --static really_inline --void setbit512(m512 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); --#if !defined(HAVE_AVX2) -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo.lo; -- } else if (n < 256) { -- sub = &ptr->lo.hi; -- } else if (n < 384) { -- sub = &ptr->hi.lo; -- } else { -- sub = &ptr->hi.hi; -- } -- setbit128(sub, n % 128); --#elif defined(HAVE_AVX512) -- *ptr = or512(mask1bit512(n), *ptr); --#else -- m256 *sub; -- if (n < 256) { -- sub = &ptr->lo; -- } else { -- sub = &ptr->hi; -- n -= 256; -- } -- setbit256(sub, n); --#endif --} -- --// switches off bit N in the given vector. --static really_inline --void clearbit512(m512 *ptr, unsigned int n) { -- assert(n < sizeof(*ptr) * 8); --#if !defined(HAVE_AVX2) -- m128 *sub; -- if (n < 128) { -- sub = &ptr->lo.lo; -- } else if (n < 256) { -- sub = &ptr->lo.hi; -- } else if (n < 384) { -- sub = &ptr->hi.lo; -- } else { -- sub = &ptr->hi.hi; -- } -- clearbit128(sub, n % 128); --#elif defined(HAVE_AVX512) -- *ptr = andnot512(mask1bit512(n), *ptr); --#else -- m256 *sub; -- if (n < 256) { -- sub = &ptr->lo; -- } else { -- sub = &ptr->hi; -- n -= 256; -- } -- clearbit256(sub, n); --#endif --} -- --// tests bit N in the given vector. --static really_inline --char testbit512(m512 val, unsigned int n) { -- assert(n < sizeof(val) * 8); --#if !defined(HAVE_AVX2) -- m128 sub; -- if (n < 128) { -- sub = val.lo.lo; -- } else if (n < 256) { -- sub = val.lo.hi; -- } else if (n < 384) { -- sub = val.hi.lo; -- } else { -- sub = val.hi.hi; -- } -- return testbit128(sub, n % 128); --#elif defined(HAVE_AVX512) -- const m512 mask = mask1bit512(n); -- return !!_mm512_test_epi8_mask(mask, val); + #if !defined(_WIN32) && !defined(CPUID_H_) ++#if defined(__x86_64__) + #include ++#endif + /* system header doesn't have a header guard */ + #define CPUID_H_ + #endif +diff --git a/src/util/cpuid_inline.h b/src/util/cpuid_inline.h +index b7b4245..b228c1d 100644 +--- a/src/util/cpuid_inline.h ++++ b/src/util/cpuid_inline.h +@@ -32,17 +32,20 @@ + #include "ue2common.h" + #include "cpuid_flags.h" + ++#if defined(__x86_64__) || defined(_M_X64) + #if !defined(_WIN32) && !defined(CPUID_H_) + #include + /* system header doesn't have a header guard */ + #define CPUID_H_ + #endif ++#endif + + #ifdef __cplusplus + extern "C" + { + #endif + ++#if defined(__x86_64__) || defined(_M_X64) + static inline + void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, + unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { +@@ -57,6 +60,7 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, + *edx = a[3]; + #endif + } ++#endif + + // ECX + #define CPUID_SSE3 (1 << 0) +@@ -93,11 +97,12 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax, + #define CPUID_XCR0_AVX512 \ + (CPUID_XCR0_OPMASK | CPUID_XCR0_ZMM_Hi256 | CPUID_XCR0_Hi16_ZMM) + ++#if defined(__x86_64__) + static inline + u64a xgetbv(u32 op) { + #if defined(_WIN32) || defined(__INTEL_COMPILER) + return _xgetbv(op); -#else -- m256 sub; -- if (n < 256) { -- sub = val.lo; -- } else { -- sub = val.hi; -- n -= 256; -- } -- return testbit256(sub, n); --#endif --} -- --#endif -diff --git a/src/util/simd_x86.h b/src/util/simd_x86.h ++#elif defined(__x86_64__) + u32 a, d; + __asm__ volatile ( + "xgetbv\n" +@@ -252,6 +257,16 @@ int check_popcnt(void) { + cpuid(1, 0, &eax, &ebx, &ecx, &edx); + return !!(ecx & CPUID_POPCNT); + } ++#endif //__x86_64__ ++ ++static inline ++int check_neon(void) { ++#if defined(__aarch64__) ++ return 1; ++#else ++ return 0; ++#endif ++} + + #ifdef __cplusplus + } /* extern "C" */ +diff --git a/src/util/intrinsics.h b/src/util/intrinsics.h +index edc4f6e..ece3b1a 100644 +--- a/src/util/intrinsics.h ++++ b/src/util/intrinsics.h +@@ -55,10 +55,22 @@ + # endif + #endif + ++#ifdef __cplusplus ++# if defined(HAVE_CXX_ARM_NEON_H) ++# define USE_ARM_NEON_H ++# endif ++#else // C ++# if defined(HAVE_C_ARM_NEON_H) ++# define USE_ARM_NEON_H ++# endif ++#endif ++ + #if defined(USE_X86INTRIN_H) + #include + #elif defined(USE_INTRIN_H) + #include ++#elif defined(USE_ARM_NEON_H) ++#include + #else + #error no intrinsics file + #endif +diff --git a/src/util/popcount.h b/src/util/popcount.h +index eb08f6b..7d794d1 100644 +--- a/src/util/popcount.h ++++ b/src/util/popcount.h +@@ -41,6 +41,8 @@ u32 popcount32(u32 x) { + #if defined(HAVE_POPCOUNT_INSTR) + // Single-instruction builtin. + return _mm_popcnt_u32(x); ++#elif defined(HAVE_NEON) ++ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x))); + #else + // Fast branch-free version from bit-twiddling hacks as older Intel + // processors do not have a POPCNT instruction. +@@ -63,7 +65,9 @@ u32 popcount64(u64a x) { + x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); + x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; + return (x * 0x0101010101010101) >> 56; +-# endif ++#endif ++#elif defined(HAVE_NEON) ++ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x))); + #else + // Synthesise from two 32-bit cases. + return popcount32(x >> 32) + popcount32(x); +diff --git a/src/util/simd_arm.h b/src/util/simd_arm.h new file mode 100644 -index 0000000..59ac642 +index 0000000..cce119f --- /dev/null -+++ b/src/util/simd_x86.h -@@ -0,0 +1,1334 @@ ++++ b/src/util/simd_arm.h +@@ -0,0 +1,1069 @@ +/* + * Copyright (c) 2015-2017, Intel Corporation ++ * 2020.01 - Use the neon instruction to implement the function of 128-bit operation. ++ * Huawei Technologies Co., Ltd. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: @@ -11488,16 +1457,12 @@ index 0000000..59ac642 + * \brief SIMD types and primitive operations. + */ + -+#ifndef SIMD_X86 -+#define SIMD_X86 -+ -+#if !defined(_WIN32) && !defined(__SSSE3__) -+#error SSSE3 instructions must be enabled -+#endif ++#ifndef SIMD_ARM ++#define SIMD_ARM + +#include "config.h" -+#include "ue2common.h" +#include "simd_types.h" ++#include "ue2common.h" +#include "unaligned.h" +#include "util/arch.h" +#include "util/intrinsics.h" @@ -11507,13 +1472,13 @@ index 0000000..59ac642 +// Define a common assume_aligned using an appropriate compiler built-in, if +// it's available. Note that we need to handle C or C++ compilation. +#ifdef __cplusplus -+# ifdef HAVE_CXX_BUILTIN_ASSUME_ALIGNED -+# define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) -+# endif ++#ifdef HAVE_CXX_BUILTIN_ASSUME_ALIGNED ++#define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) ++#endif +#else -+# ifdef HAVE_CC_BUILTIN_ASSUME_ALIGNED -+# define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) -+# endif ++#ifdef HAVE_CC_BUILTIN_ASSUME_ALIGNED ++#define assume_aligned(x, y) __builtin_assume_aligned((x), (y)) ++#endif +#endif + +// Fallback to identity case. @@ -11529,29 +1494,34 @@ index 0000000..59ac642 +} +#endif + -+static really_inline m128 ones128(void) { -+#if defined(__GNUC__) || defined(__INTEL_COMPILER) -+ /* gcc gets this right */ -+ return _mm_set1_epi8(0xFF); -+#else -+ /* trick from Intel's optimization guide to generate all-ones. -+ * ICC converts this to the single cmpeq instruction */ -+ return _mm_cmpeq_epi8(_mm_setzero_si128(), _mm_setzero_si128()); -+#endif ++/* ++** extend 4.8.5 neon inline assembly functions ++*/ ++__extension__ static __inline uint64x2_t __attribute__((__always_inline__)) ++vmvnq_u64(uint64x2_t a) { ++ uint64x2_t result; ++ __asm__("mvn %0.16b,%1.16b" : "=w"(result) : "w"(a) : /* No clobbers */); ++ return result; +} + -+static really_inline m128 zeroes128(void) { -+ return _mm_setzero_si128(); ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wshadow" ++ ++static really_inline m128 ones128(void) { ++ m128 result; ++ result.vect_s32 = vdupq_n_s32(0xFFFFFFFF); ++ return result; +} + -+/** \brief Bitwise not for m128*/ -+static really_inline m128 not128(m128 a) { -+ return _mm_xor_si128(a, ones128()); ++static really_inline m128 zeroes128(void) { ++ m128 result; ++ result.vect_s32 = vdupq_n_s32(0x0); ++ return result; +} + +/** \brief Return 1 if a and b are different otherwise 0 */ +static really_inline int diff128(m128 a, m128 b) { -+ return (_mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) ^ 0xffff); ++ return !!vaddlvq_s16(veorq_s16(a.vect_s16, b.vect_s16)); +} + +static really_inline int isnonzero128(m128 a) { @@ -11563,8 +1533,12 @@ index 0000000..59ac642 + * mask indicating which 32-bit words contain differences. + */ +static really_inline u32 diffrich128(m128 a, m128 b) { -+ a = _mm_cmpeq_epi32(a, b); -+ return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0xf; ++ m128 tmp; ++ tmp.vect_u32 = vmvnq_u32(vceqq_u32(a.vect_u32, b.vect_u32)); ++ return ((vgetq_lane_u32(tmp.vect_u32, 3) & 0x8) | ++ (vgetq_lane_u32(tmp.vect_u32, 2) & 0x4) | ++ (vgetq_lane_u32(tmp.vect_u32, 1) & 0x2) | ++ (vgetq_lane_u32(tmp.vect_u32, 0) & 0x1)); +} + +/** @@ -11572,108 +1546,167 @@ index 0000000..59ac642 + * returns a 4-bit mask indicating which 64-bit words contain differences. + */ +static really_inline u32 diffrich64_128(m128 a, m128 b) { -+#if defined(HAVE_SSE41) -+ a = _mm_cmpeq_epi64(a, b); -+ return ~(_mm_movemask_ps(_mm_castsi128_ps(a))) & 0x5; -+#else -+ u32 d = diffrich128(a, b); -+ return (d | (d >> 1)) & 0x5; -+#endif ++ m128 tmp; ++ tmp.vect_u64 = vmvnq_u64(vceqq_u64(a.vect_u64, b.vect_u64)); ++ return (u32)((vgetq_lane_u64(tmp.vect_u64, 1) & 0x4) | ++ (vgetq_lane_u64(tmp.vect_u64, 0) & 0x1)); ++} ++ ++static really_really_inline m128 lshift64_m128(m128 a, unsigned b) { ++ assert(b <= 63); ++ m128 result; ++ result.vect_s64 = vshlq_n_s64(a.vect_s64, b); ++ return result; +} + -+static really_really_inline -+m128 lshift64_m128(m128 a, unsigned b) { -+#if defined(HAVE__BUILTIN_CONSTANT_P) -+ if (__builtin_constant_p(b)) { -+ return _mm_slli_epi64(a, b); ++static really_really_inline m128 rshift64_m128(m128 a, int imm8) { ++ assert(imm8 >= 0 && imm8 <= 63); ++ if (unlikely(imm8 == 0)) { ++ return a; + } -+#endif -+ m128 x = _mm_cvtsi32_si128(b); -+ return _mm_sll_epi64(a, x); ++ m128 result; ++ result.vect_u64 = vshrq_n_u64(a.vect_u64, imm8); ++ return result; +} + -+#define rshift64_m128(a, b) _mm_srli_epi64((a), (b)) -+#define eq128(a, b) _mm_cmpeq_epi8((a), (b)) -+#define movemask128(a) ((u32)_mm_movemask_epi8((a))) ++static really_really_inline m128 eq128(m128 a, m128 b) { ++ m128 result; ++ result.vect_u8 = vceqq_s8(a.vect_s8, b.vect_s8); ++ return result; ++} ++ ++static really_really_inline u32 movemask128(m128 a) { ++ m128 result; ++ result.vect_u8 = vshrq_n_u8(a.vect_u8, 7); ++ result.vect_u16 = vsraq_n_u16(result.vect_u16, result.vect_u16, 7); ++ result.vect_u32 = vsraq_n_u32(result.vect_u32, result.vect_u32, 14); ++ result.vect_u64 = vsraq_n_u64(result.vect_u64, result.vect_u64, 28); ++ return (u32)(vgetq_lane_u8(result.vect_u8, 0) | ++ ((u32)vgetq_lane_u8(result.vect_u8, 8) << 8)); ++} ++ ++static really_really_inline m128 rshiftbyte_m128(m128 a, int imm8) { ++ assert(imm8 >= 0 && imm8 <= 15); ++ m128 result; ++ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0), imm8); ++ return result; ++} ++ ++static really_really_inline m128 lshiftbyte_m128(m128 a, int imm8) { ++ assert(imm8 >= 0 && imm8 <= 15); ++ m128 result; ++ if (unlikely(imm8 == 0)) { ++ return a; ++ } ++ result.vect_s8 = vextq_s8(vdupq_n_s8(0), a.vect_s8, (16 - imm8)); ++ return result; ++} + +static really_inline m128 set16x8(u8 c) { -+ return _mm_set1_epi8(c); ++ m128 result; ++ result.vect_s8 = vdupq_n_s8(c); ++ return result; +} + +static really_inline m128 set4x32(u32 c) { -+ return _mm_set1_epi32(c); ++ m128 result; ++ result.vect_s32 = vdupq_n_s32(c); ++ return result; +} + +static really_inline m128 set2x64(u64a c) { -+ return _mm_set1_epi32(c); ++ m128 result; ++ result.vect_u64 = vdupq_n_u64(c); ++ return result; +} + +static really_inline u32 movd(const m128 in) { -+ return _mm_cvtsi128_si32(in); ++ u32 result; ++ result = vgetq_lane_u32(in.vect_u32, 0); ++ return result; +} + +static really_inline u64a movq(const m128 in) { -+#if defined(ARCH_X86_64) -+ return _mm_cvtsi128_si64(in); -+#else // 32-bit - this is horrific -+ u32 lo = movd(in); -+ u32 hi = movd(_mm_srli_epi64(in, 32)); -+ return (u64a)hi << 32 | lo; -+#endif ++ return vgetq_lane_u64(in.vect_u64, 0); +} + +/* another form of movq */ -+static really_inline -+m128 load_m128_from_u64a(const u64a *p) { -+ return _mm_set_epi64x(0LL, *p); ++static really_inline m128 load_m128_from_u64a(const u64a *p) { ++ m128 result; ++ __asm__ __volatile__("ldr %d0, %1 \n\t" ++ : "=w"(result) ++ : "Utv"(*p) ++ : /* No clobbers */ ++ ); ++ return result; ++} ++ ++/*The x86 platform does not perform the lower 2 bit operation. ++If the value of imm exceeds 2 bit, a compilation error occurs.*/ ++static really_inline u32 extract32from128(m128 a, int imm) { ++ return vgetq_lane_s32(a.vect_s32, imm & 0x0003); ++} ++ ++/*The x86 platform does not perform the lower 1 bit operation. ++If the value of imm exceeds 1 bit, a compilation error occurs.*/ ++static really_inline u64a extract64from128(m128 a, int imm) { ++ return vgetq_lane_s64(a.vect_s64, imm & 0x0001); +} + -+#define rshiftbyte_m128(a, count_immed) _mm_srli_si128(a, count_immed) -+#define lshiftbyte_m128(a, count_immed) _mm_slli_si128(a, count_immed) -+ -+#if defined(HAVE_SSE41) -+#define extract32from128(a, imm) _mm_extract_epi32(a, imm) -+#define extract64from128(a, imm) _mm_extract_epi64(a, imm) -+#else -+#define extract32from128(a, imm) movd(_mm_srli_si128(a, imm << 2)) -+#define extract64from128(a, imm) movq(_mm_srli_si128(a, imm << 3)) -+#endif -+ -+#if !defined(HAVE_AVX2) -+// TODO: this entire file needs restructuring - this carveout is awful +#define extractlow64from256(a) movq(a.lo) +#define extractlow32from256(a) movd(a.lo) -+#if defined(HAVE_SSE41) -+#define extract32from256(a, imm) _mm_extract_epi32((imm >> 2) ? a.hi : a.lo, imm % 4) -+#define extract64from256(a, imm) _mm_extract_epi64((imm >> 1) ? a.hi : a.lo, imm % 2) -+#else -+#define extract32from256(a, imm) movd(_mm_srli_si128((imm >> 2) ? a.hi : a.lo, (imm % 4) * 4)) -+#define extract64from256(a, imm) movq(_mm_srli_si128((imm >> 1) ? a.hi : a.lo, (imm % 2) * 8)) -+#endif + -+#endif // !AVX2 ++/*The x86 platform does not perform the lower 2 bit operation. ++If the value of imm exceeds 2 bit, a compilation error occurs.*/ ++static really_inline u32 extract32from256(m256 a, int imm) { ++ return vgetq_lane_s32((imm >> 2) ? a.hi.vect_s32 : a.lo.vect_s32, ++ imm & 0x0003); ++} ++ ++/*The x86 platform does not perform the lower 1 bit operation. ++If the value of imm exceeds 1 bit, a compilation error occurs.*/ ++static really_inline u64a extract64from256(m256 a, int imm) { ++ return vgetq_lane_s64((imm >> 1) ? a.hi.vect_s64 : a.lo.vect_s64, ++ imm & 0x0001); ++} + +static really_inline m128 and128(m128 a, m128 b) { -+ return _mm_and_si128(a,b); ++ m128 result; ++ result.vect_s32 = vandq_s32(a.vect_s32, b.vect_s32); ++ return result; ++} ++ ++static really_inline m128 not128(m128 a) { ++ m128 result; ++ result.vect_s32 = vmvnq_s32(a.vect_s32); ++ return result; +} + +static really_inline m128 xor128(m128 a, m128 b) { -+ return _mm_xor_si128(a,b); ++ m128 result; ++ result.vect_s32 = veorq_s32(a.vect_s32, b.vect_s32); ++ return result; +} + +static really_inline m128 or128(m128 a, m128 b) { -+ return _mm_or_si128(a,b); ++ m128 result; ++ result.vect_s32 = vorrq_s32(a.vect_s32, b.vect_s32); ++ return result; +} + +static really_inline m128 andnot128(m128 a, m128 b) { -+ return _mm_andnot_si128(a, b); ++ m128 result; ++ result.vect_s32 = vbicq_s32(b.vect_s32, a.vect_s32); ++ return result; +} + +// aligned load +static really_inline m128 load128(const void *ptr) { + assert(ISALIGNED_N(ptr, alignof(m128))); + ptr = assume_aligned(ptr, 16); -+ return _mm_load_si128((const m128 *)ptr); ++ m128 result; ++ result.vect_s32 = vld1q_s32((const int32_t *)ptr); ++ return result; +} + +// aligned store @@ -11685,24 +1718,24 @@ index 0000000..59ac642 + +// unaligned load +static really_inline m128 loadu128(const void *ptr) { -+ return _mm_loadu_si128((const m128 *)ptr); ++ m128 result; ++ result.vect_s32 = vld1q_s32((const int32_t *)ptr); ++ return result; +} + +// unaligned store +static really_inline void storeu128(void *ptr, m128 a) { -+ _mm_storeu_si128 ((m128 *)ptr, a); ++ vst1q_s32((int32_t *)ptr, a.vect_s32); +} + +// packed unaligned store of first N bytes -+static really_inline -+void storebytes128(void *ptr, m128 a, unsigned int n) { ++static really_inline void storebytes128(void *ptr, m128 a, unsigned int n) { + assert(n <= sizeof(a)); + memcpy(ptr, &a, n); +} + +// packed unaligned load of first N bytes, pad with zero -+static really_inline -+m128 loadbytes128(const void *ptr, unsigned int n) { ++static really_inline m128 loadbytes128(const void *ptr, unsigned int n) { + m128 a = zeroes128(); + assert(n <= sizeof(a)); + memcpy(&a, ptr, n); @@ -11717,8 +1750,7 @@ index 0000000..59ac642 +} +#endif + -+static really_inline -+m128 mask1bit128(unsigned int n) { ++static really_inline m128 mask1bit128(unsigned int n) { + assert(n < sizeof(m128) * 8); + u32 mask_idx = ((n % 8) * 64) + 95; + mask_idx -= n / 8; @@ -11726,275 +1758,191 @@ index 0000000..59ac642 +} + +// switches on bit N in the given vector. -+static really_inline -+void setbit128(m128 *ptr, unsigned int n) { ++static really_inline void setbit128(m128 *ptr, unsigned int n) { + *ptr = or128(mask1bit128(n), *ptr); +} + +// switches off bit N in the given vector. -+static really_inline -+void clearbit128(m128 *ptr, unsigned int n) { ++static really_inline void clearbit128(m128 *ptr, unsigned int n) { + *ptr = andnot128(mask1bit128(n), *ptr); +} + +// tests bit N in the given vector. -+static really_inline -+char testbit128(m128 val, unsigned int n) { ++static really_inline char testbit128(m128 val, unsigned int n) { + const m128 mask = mask1bit128(n); -+#if defined(HAVE_SSE41) -+ return !_mm_testz_si128(mask, val); -+#else + return isnonzero128(and128(mask, val)); -+#endif +} + +// offset must be an immediate -+#define palignr(r, l, offset) _mm_alignr_epi8(r, l, offset) ++/*The x86 platform does not perform the lower 8 bit operation. ++If the value of imm exceeds 8 bit, a compilation error occurs.*/ ++static really_inline m128 palignr(m128 a, m128 b, int count) { ++ m128 result; ++ count = count & 0xff; ++ if (likely(count < 16)) { ++ result.vect_s8 = vextq_s8(b.vect_s8, a.vect_s8, count); ++ } else if (count < 32) { ++ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0x0), count - 16); ++ } else { ++ result.vect_s32 = vdupq_n_s32(0); ++ } ++ return result; ++} + -+static really_inline -+m128 pshufb_m128(m128 a, m128 b) { ++static really_inline m128 pshufb_m128(m128 a, m128 b) { + m128 result; -+ result = _mm_shuffle_epi8(a, b); ++ __asm__ __volatile__("movi v3.16b, 0x8f \n\t" ++ "and v3.16b, v3.16b, %2.16b \n\t" ++ "tbl %0.16b, {%1.16b}, v3.16b \n\t" ++ : "=w"(result) ++ : "w"(a), "w"(b) ++ : "v3"); + return result; +} + -+static really_inline -+m256 pshufb_m256(m256 a, m256 b) { -+#if defined(HAVE_AVX2) -+ return _mm256_shuffle_epi8(a, b); -+#else ++static really_inline m256 pshufb_m256(m256 a, m256 b) { + m256 rv; + rv.lo = pshufb_m128(a.lo, b.lo); + rv.hi = pshufb_m128(a.hi, b.hi); + return rv; -+#endif -+} -+ -+#if defined(HAVE_AVX512) -+static really_inline -+m512 pshufb_m512(m512 a, m512 b) { -+ return _mm512_shuffle_epi8(a, b); +} + -+static really_inline -+m512 maskz_pshufb_m512(__mmask64 k, m512 a, m512 b) { -+ return _mm512_maskz_shuffle_epi8(k, a, b); -+} -+#endif -+ -+static really_inline -+m128 variable_byte_shift_m128(m128 in, s32 amount) { ++static really_inline m128 variable_byte_shift_m128(m128 in, s32 amount) { + assert(amount >= -16 && amount <= 16); + m128 shift_mask = loadu128(vbs_mask_data + 16 - amount); + return pshufb_m128(in, shift_mask); +} + -+static really_inline -+m128 max_u8_m128(m128 a, m128 b) { -+ return _mm_max_epu8(a, b); ++static really_inline m128 max_u8_m128(m128 a, m128 b) { ++ m128 result; ++ result.vect_u8 = vmaxq_u8(a.vect_u8, b.vect_u8); ++ return result; +} + -+static really_inline -+m128 min_u8_m128(m128 a, m128 b) { -+ return _mm_min_epu8(a, b); ++static really_inline m128 min_u8_m128(m128 a, m128 b) { ++ m128 result; ++ result.vect_u8 = vminq_u8(a.vect_u8, b.vect_u8); ++ return result; +} + -+static really_inline -+m128 sadd_u8_m128(m128 a, m128 b) { -+ return _mm_adds_epu8(a, b); ++static really_inline m128 sadd_u8_m128(m128 a, m128 b) { ++ m128 result; ++ result.vect_u8 = vqaddq_u8(a.vect_u8, b.vect_u8); ++ return result; +} + -+static really_inline -+m128 sub_u8_m128(m128 a, m128 b) { -+ return _mm_sub_epi8(a, b); ++static really_inline m128 sub_u8_m128(m128 a, m128 b) { ++ m128 result; ++ result.vect_u8 = vsubq_u8(a.vect_u8, b.vect_u8); ++ return result; +} + -+static really_inline -+m128 set64x2(u64a hi, u64a lo) { -+ return _mm_set_epi64x(hi, lo); ++static really_inline m128 set64x2(int64_t hi, int64_t lo) { ++ m128 result; ++ result.vect_s64 = vsetq_lane_s64(hi, vdupq_n_s64(lo), 1); ++ return result; +} + -+static really_inline -+m128 set32x4(int i3, int i2, int i1, int i0) { -+ return _mm_set_epi32(i3, i2, i1, i0); ++static really_inline m128 set32x4(int i3, int i2, int i1, int i0) { ++ m128 result; ++ result.vect_s32 = vsetq_lane_s32( ++ i3, vsetq_lane_s32(i2, vsetq_lane_s32(i1, vdupq_n_s32(i0), 1), 2), 3); ++ return result; +} + +/**** + **** 256-bit Primitives + ****/ + -+#if defined(HAVE_AVX2) -+ -+static really_really_inline -+m256 lshift64_m256(m256 a, unsigned b) { -+#if defined(HAVE__BUILTIN_CONSTANT_P) -+ if (__builtin_constant_p(b)) { -+ return _mm256_slli_epi64(a, b); -+ } -+#endif -+ m128 x = _mm_cvtsi32_si128(b); -+ return _mm256_sll_epi64(a, x); -+} -+ -+#define rshift64_m256(a, b) _mm256_srli_epi64((a), (b)) -+ -+static really_inline -+m256 set32x8(u32 in) { -+ return _mm256_set1_epi8(in); -+} -+ -+#define eq256(a, b) _mm256_cmpeq_epi8((a), (b)) -+#define movemask256(a) ((u32)_mm256_movemask_epi8((a))) -+ -+static really_inline -+m256 set2x128(m128 a) { -+ return _mm256_broadcastsi128_si256(a); -+} -+ -+#else -+ -+static really_really_inline -+m256 lshift64_m256(m256 a, int b) { ++static really_really_inline m256 lshift64_m256(m256 a, int b) { + m256 rv = a; + rv.lo = lshift64_m128(rv.lo, b); + rv.hi = lshift64_m128(rv.hi, b); + return rv; +} + -+static really_inline -+m256 rshift64_m256(m256 a, int b) { ++static really_inline m256 rshift64_m256(m256 a, int b) { + m256 rv = a; + rv.lo = rshift64_m128(rv.lo, b); + rv.hi = rshift64_m128(rv.hi, b); + return rv; +} -+static really_inline -+m256 set32x8(u32 in) { ++static really_inline m256 set32x8(u32 in) { + m256 rv; -+ rv.lo = set16x8((u8) in); ++ rv.lo = set16x8((u8)in); + rv.hi = rv.lo; + return rv; +} + -+static really_inline -+m256 eq256(m256 a, m256 b) { ++static really_inline m256 eq256(m256 a, m256 b) { + m256 rv; + rv.lo = eq128(a.lo, b.lo); + rv.hi = eq128(a.hi, b.hi); + return rv; +} + -+static really_inline -+u32 movemask256(m256 a) { ++static really_inline u32 movemask256(m256 a) { + u32 lo_mask = movemask128(a.lo); + u32 hi_mask = movemask128(a.hi); + return lo_mask | (hi_mask << 16); +} + -+static really_inline -+m256 set2x128(m128 a) { ++static really_inline m256 set2x128(m128 a) { + m256 rv = {a, a}; + return rv; +} -+#endif + +static really_inline m256 zeroes256(void) { -+#if defined(HAVE_AVX2) -+ return _mm256_setzero_si256(); -+#else + m256 rv = {zeroes128(), zeroes128()}; + return rv; -+#endif +} + +static really_inline m256 ones256(void) { -+#if defined(HAVE_AVX2) -+ m256 rv = _mm256_set1_epi8(0xFF); -+#else + m256 rv = {ones128(), ones128()}; -+#endif + return rv; +} + -+#if defined(HAVE_AVX2) -+static really_inline m256 and256(m256 a, m256 b) { -+ return _mm256_and_si256(a, b); -+} -+#else +static really_inline m256 and256(m256 a, m256 b) { + m256 rv; + rv.lo = and128(a.lo, b.lo); + rv.hi = and128(a.hi, b.hi); + return rv; +} -+#endif + -+#if defined(HAVE_AVX2) -+static really_inline m256 or256(m256 a, m256 b) { -+ return _mm256_or_si256(a, b); -+} -+#else +static really_inline m256 or256(m256 a, m256 b) { + m256 rv; + rv.lo = or128(a.lo, b.lo); + rv.hi = or128(a.hi, b.hi); + return rv; +} -+#endif + -+#if defined(HAVE_AVX2) -+static really_inline m256 xor256(m256 a, m256 b) { -+ return _mm256_xor_si256(a, b); -+} -+#else +static really_inline m256 xor256(m256 a, m256 b) { + m256 rv; + rv.lo = xor128(a.lo, b.lo); + rv.hi = xor128(a.hi, b.hi); + return rv; +} -+#endif + -+#if defined(HAVE_AVX2) -+static really_inline m256 not256(m256 a) { -+ return _mm256_xor_si256(a, ones256()); -+} -+#else +static really_inline m256 not256(m256 a) { + m256 rv; + rv.lo = not128(a.lo); + rv.hi = not128(a.hi); + return rv; +} -+#endif + -+#if defined(HAVE_AVX2) -+static really_inline m256 andnot256(m256 a, m256 b) { -+ return _mm256_andnot_si256(a, b); -+} -+#else +static really_inline m256 andnot256(m256 a, m256 b) { + m256 rv; + rv.lo = andnot128(a.lo, b.lo); + rv.hi = andnot128(a.hi, b.hi); + return rv; +} -+#endif + +static really_inline int diff256(m256 a, m256 b) { -+#if defined(HAVE_AVX2) -+ return !!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(a, b)) ^ (int)-1); -+#else + return diff128(a.lo, b.lo) || diff128(a.hi, b.hi); -+#endif +} + +static really_inline int isnonzero256(m256 a) { -+#if defined(HAVE_AVX2) -+ return !!diff256(a, zeroes256()); -+#else + return isnonzero128(or128(a.lo, a.hi)); -+#endif +} + +/** @@ -12002,16 +1950,23 @@ index 0000000..59ac642 + * mask indicating which 32-bit words contain differences. + */ +static really_inline u32 diffrich256(m256 a, m256 b) { -+#if defined(HAVE_AVX2) -+ a = _mm256_cmpeq_epi32(a, b); -+ return ~(_mm256_movemask_ps(_mm256_castsi256_ps(a))) & 0xFF; -+#else -+ m128 z = zeroes128(); -+ a.lo = _mm_cmpeq_epi32(a.lo, b.lo); -+ a.hi = _mm_cmpeq_epi32(a.hi, b.hi); -+ m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.hi), z); -+ return ~(_mm_movemask_epi8(packed)) & 0xff; -+#endif ++ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32); ++ uint32x4_t y = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32); ++ uint8x8_t lo = vqmovn_u16(vcombine_u16(vqmovn_u32(x), vqmovn_u32(y))); ++ ++ static const int8_t __attribute__((aligned(16))) ++ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; ++ uint8x8_t mask_and = vdup_n_u8(0x80); ++ int8x8_t mask_shift = vld1_s8(xr); ++ ++ lo = vand_u8(lo, mask_and); ++ lo = vshl_u8(lo, mask_shift); ++ ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); ++ ++ return ~(lo[0] & 0xFF) & 0xff; +} + +/** @@ -12026,24 +1981,16 @@ index 0000000..59ac642 +// aligned load +static really_inline m256 load256(const void *ptr) { + assert(ISALIGNED_N(ptr, alignof(m256))); -+#if defined(HAVE_AVX2) -+ return _mm256_load_si256((const m256 *)ptr); -+#else -+ m256 rv = { load128(ptr), load128((const char *)ptr + 16) }; ++ m256 rv = {load128(ptr), load128((const char *)ptr + 16)}; + return rv; -+#endif +} + +// aligned load of 128-bit value to low and high part of 256-bit value +static really_inline m256 load2x128(const void *ptr) { -+#if defined(HAVE_AVX2) -+ return set2x128(load128(ptr)); -+#else + assert(ISALIGNED_N(ptr, alignof(m128))); + m256 rv; + rv.hi = rv.lo = load128(ptr); + return rv; -+#endif +} + +static really_inline m256 loadu2x128(const void *ptr) { @@ -12053,74 +2000,52 @@ index 0000000..59ac642 +// aligned store +static really_inline void store256(void *ptr, m256 a) { + assert(ISALIGNED_N(ptr, alignof(m256))); -+#if defined(HAVE_AVX2) -+ _mm256_store_si256((m256 *)ptr, a); -+#else + ptr = assume_aligned(ptr, 16); + *(m256 *)ptr = a; -+#endif +} + +// unaligned load +static really_inline m256 loadu256(const void *ptr) { -+#if defined(HAVE_AVX2) -+ return _mm256_loadu_si256((const m256 *)ptr); -+#else -+ m256 rv = { loadu128(ptr), loadu128((const char *)ptr + 16) }; ++ m256 rv = {loadu128(ptr), loadu128((const char *)ptr + 16)}; + return rv; -+#endif +} + +// unaligned store +static really_inline void storeu256(void *ptr, m256 a) { -+#if defined(HAVE_AVX2) -+ _mm256_storeu_si256((m256 *)ptr, a); -+#else + storeu128(ptr, a.lo); + storeu128((char *)ptr + 16, a.hi); -+#endif +} + +// packed unaligned store of first N bytes -+static really_inline -+void storebytes256(void *ptr, m256 a, unsigned int n) { ++static really_inline void storebytes256(void *ptr, m256 a, unsigned int n) { + assert(n <= sizeof(a)); + memcpy(ptr, &a, n); +} + +// packed unaligned load of first N bytes, pad with zero -+static really_inline -+m256 loadbytes256(const void *ptr, unsigned int n) { ++static really_inline m256 loadbytes256(const void *ptr, unsigned int n) { + m256 a = zeroes256(); + assert(n <= sizeof(a)); + memcpy(&a, ptr, n); + return a; +} + -+static really_inline -+m256 mask1bit256(unsigned int n) { ++static really_inline m256 mask1bit256(unsigned int n) { + assert(n < sizeof(m256) * 8); + u32 mask_idx = ((n % 8) * 64) + 95; + mask_idx -= n / 8; + return loadu256(&simd_onebit_masks[mask_idx]); +} + -+static really_inline -+m256 set64x4(u64a hi_1, u64a hi_0, u64a lo_1, u64a lo_0) { -+#if defined(HAVE_AVX2) -+ return _mm256_set_epi64x(hi_1, hi_0, lo_1, lo_0); -+#else ++static really_inline m256 set64x4(u64a hi_1, u64a hi_0, u64a lo_1, u64a lo_0) { + m256 rv; + rv.hi = set64x2(hi_1, hi_0); + rv.lo = set64x2(lo_1, lo_0); + return rv; -+#endif +} + -+#if !defined(HAVE_AVX2) +// switches on bit N in the given vector. -+static really_inline -+void setbit256(m256 *ptr, unsigned int n) { ++static really_inline void setbit256(m256 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); + m128 *sub; + if (n < 128) { @@ -12133,8 +2058,7 @@ index 0000000..59ac642 +} + +// switches off bit N in the given vector. -+static really_inline -+void clearbit256(m256 *ptr, unsigned int n) { ++static really_inline void clearbit256(m256 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); + m128 *sub; + if (n < 128) { @@ -12147,8 +2071,7 @@ index 0000000..59ac642 +} + +// tests bit N in the given vector. -+static really_inline -+char testbit256(m256 val, unsigned int n) { ++static really_inline char testbit256(m256 val, unsigned int n) { + assert(n < sizeof(val) * 8); + m128 sub; + if (n < 128) { @@ -12160,85 +2083,15 @@ index 0000000..59ac642 + return testbit128(sub, n); +} + -+static really_really_inline -+m128 movdq_hi(m256 x) { -+ return x.hi; -+} ++static really_really_inline m128 movdq_hi(m256 x) { return x.hi; } + -+static really_really_inline -+m128 movdq_lo(m256 x) { -+ return x.lo; -+} ++static really_really_inline m128 movdq_lo(m256 x) { return x.lo; } + -+static really_inline -+m256 combine2x128(m128 hi, m128 lo) { ++static really_inline m256 combine2x128(m128 hi, m128 lo) { + m256 rv = {lo, hi}; + return rv; +} + -+#else // AVX2 -+ -+// switches on bit N in the given vector. -+static really_inline -+void setbit256(m256 *ptr, unsigned int n) { -+ *ptr = or256(mask1bit256(n), *ptr); -+} -+ -+static really_inline -+void clearbit256(m256 *ptr, unsigned int n) { -+ *ptr = andnot256(mask1bit256(n), *ptr); -+} -+ -+// tests bit N in the given vector. -+static really_inline -+char testbit256(m256 val, unsigned int n) { -+ const m256 mask = mask1bit256(n); -+ return !_mm256_testz_si256(mask, val); -+} -+ -+static really_really_inline -+m128 movdq_hi(m256 x) { -+ return _mm256_extracti128_si256(x, 1); -+} -+ -+static really_really_inline -+m128 movdq_lo(m256 x) { -+ return _mm256_extracti128_si256(x, 0); -+} -+ -+#define cast256to128(a) _mm256_castsi256_si128(a) -+#define cast128to256(a) _mm256_castsi128_si256(a) -+#define swap128in256(a) _mm256_permute4x64_epi64(a, 0x4E) -+#define insert128to256(a, b, imm) _mm256_inserti128_si256(a, b, imm) -+#define rshift128_m256(a, count_immed) _mm256_srli_si256(a, count_immed) -+#define lshift128_m256(a, count_immed) _mm256_slli_si256(a, count_immed) -+#define extract64from256(a, imm) _mm_extract_epi64(_mm256_extracti128_si256(a, imm >> 1), imm % 2) -+#define extract32from256(a, imm) _mm_extract_epi32(_mm256_extracti128_si256(a, imm >> 2), imm % 4) -+#define extractlow64from256(a) _mm_cvtsi128_si64(cast256to128(a)) -+#define extractlow32from256(a) movd(cast256to128(a)) -+#define interleave256hi(a, b) _mm256_unpackhi_epi8(a, b) -+#define interleave256lo(a, b) _mm256_unpacklo_epi8(a, b) -+#define vpalignr(r, l, offset) _mm256_alignr_epi8(r, l, offset) -+ -+static really_inline -+m256 combine2x128(m128 hi, m128 lo) { -+#if defined(_mm256_set_m128i) -+ return _mm256_set_m128i(hi, lo); -+#else -+ return insert128to256(cast128to256(lo), hi, 1); -+#endif -+} -+#endif //AVX2 -+ -+#if defined(HAVE_AVX512) -+#define extract128from512(a, imm) _mm512_extracti32x4_epi32(a, imm) -+#define interleave512hi(a, b) _mm512_unpackhi_epi8(a, b) -+#define interleave512lo(a, b) _mm512_unpacklo_epi8(a, b) -+#define set2x256(a) _mm512_broadcast_i64x4(a) -+#define mask_set2x256(src, k, a) _mm512_mask_broadcast_i64x4(src, k, a) -+#define vpermq512(idx, a) _mm512_permutexvar_epi64(idx, a) -+#endif -+ +/**** + **** 384-bit Primitives + ****/ @@ -12281,8 +2134,7 @@ index 0000000..59ac642 + return rv; +} + -+static really_really_inline -+m384 lshift64_m384(m384 a, unsigned b) { ++static really_really_inline m384 lshift64_m384(m384 a, unsigned b) { + m384 rv; + rv.lo = lshift64_m128(a.lo, b); + rv.mid = lshift64_m128(a.mid, b); @@ -12295,31 +2147,57 @@ index 0000000..59ac642 + return rv; +} + -+static really_inline m384 ones384(void) { -+ m384 rv = {ones128(), ones128(), ones128()}; -+ return rv; -+} ++static really_inline m384 ones384(void) { ++ m384 rv = {ones128(), ones128(), ones128()}; ++ return rv; ++} ++ ++static really_inline int diff384(m384 a, m384 b) { ++ return diff128(a.lo, b.lo) || diff128(a.mid, b.mid) || diff128(a.hi, b.hi); ++} ++ ++static really_inline int isnonzero384(m384 a) { ++ return isnonzero128(or128(or128(a.lo, a.mid), a.hi)); ++} ++ ++/** ++ * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit ++ * mask indicating which 32-bit words contain differences. ++ */ ++static really_inline u32 diffrich384(m384 a, m384 b) { ++ m128 z = zeroes128(); ++ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32); ++ uint32x4_t y = vceqq_s32(a.mid.vect_s32, b.mid.vect_s32); ++ uint32x4_t w = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32); ++ ++ uint16x8_t q = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y)); ++ uint16x8_t p = vcombine_u16(vqmovn_u32(w), vqmovn_u32(z.vect_u32)); ++ ++ uint8x16_t input = vcombine_u8(vqmovn_u16(q), vqmovn_u16(p)); ++ ++ static const int8_t __attribute__((aligned(16))) ++ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; ++ uint8x8_t mask_and = vdup_n_u8(0x80); ++ int8x8_t mask_shift = vld1_s8(xr); ++ ++ uint8x8_t lo = vget_low_u8(input); ++ uint8x8_t hi = vget_high_u8(input); ++ ++ lo = vand_u8(lo, mask_and); ++ lo = vshl_u8(lo, mask_shift); ++ ++ hi = vand_u8(hi, mask_and); ++ hi = vshl_u8(hi, mask_shift); + -+static really_inline int diff384(m384 a, m384 b) { -+ return diff128(a.lo, b.lo) || diff128(a.mid, b.mid) || diff128(a.hi, b.hi); -+} ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); + -+static really_inline int isnonzero384(m384 a) { -+ return isnonzero128(or128(or128(a.lo, a.mid), a.hi)); -+} ++ hi = vpadd_u8(hi, hi); ++ hi = vpadd_u8(hi, hi); ++ hi = vpadd_u8(hi, hi); + -+/** -+ * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit -+ * mask indicating which 32-bit words contain differences. -+ */ -+static really_inline u32 diffrich384(m384 a, m384 b) { -+ m128 z = zeroes128(); -+ a.lo = _mm_cmpeq_epi32(a.lo, b.lo); -+ a.mid = _mm_cmpeq_epi32(a.mid, b.mid); -+ a.hi = _mm_cmpeq_epi32(a.hi, b.hi); -+ m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo, a.mid), -+ _mm_packs_epi32(a.hi, z)); -+ return ~(_mm_movemask_epi8(packed)) & 0xfff; ++ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xfff; +} + +/** @@ -12334,8 +2212,8 @@ index 0000000..59ac642 +// aligned load +static really_inline m384 load384(const void *ptr) { + assert(ISALIGNED_16(ptr)); -+ m384 rv = { load128(ptr), load128((const char *)ptr + 16), -+ load128((const char *)ptr + 32) }; ++ m384 rv = {load128(ptr), load128((const char *)ptr + 16), ++ load128((const char *)ptr + 32)}; + return rv; +} + @@ -12348,21 +2226,19 @@ index 0000000..59ac642 + +// unaligned load +static really_inline m384 loadu384(const void *ptr) { -+ m384 rv = { loadu128(ptr), loadu128((const char *)ptr + 16), -+ loadu128((const char *)ptr + 32)}; ++ m384 rv = {loadu128(ptr), loadu128((const char *)ptr + 16), ++ loadu128((const char *)ptr + 32)}; + return rv; +} + +// packed unaligned store of first N bytes -+static really_inline -+void storebytes384(void *ptr, m384 a, unsigned int n) { ++static really_inline void storebytes384(void *ptr, m384 a, unsigned int n) { + assert(n <= sizeof(a)); + memcpy(ptr, &a, n); +} + +// packed unaligned load of first N bytes, pad with zero -+static really_inline -+m384 loadbytes384(const void *ptr, unsigned int n) { ++static really_inline m384 loadbytes384(const void *ptr, unsigned int n) { + m384 a = zeroes384(); + assert(n <= sizeof(a)); + memcpy(&a, ptr, n); @@ -12370,8 +2246,7 @@ index 0000000..59ac642 +} + +// switches on bit N in the given vector. -+static really_inline -+void setbit384(m384 *ptr, unsigned int n) { ++static really_inline void setbit384(m384 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); + m128 *sub; + if (n < 128) { @@ -12385,8 +2260,7 @@ index 0000000..59ac642 +} + +// switches off bit N in the given vector. -+static really_inline -+void clearbit384(m384 *ptr, unsigned int n) { ++static really_inline void clearbit384(m384 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); + m128 *sub; + if (n < 128) { @@ -12400,8 +2274,7 @@ index 0000000..59ac642 +} + +// tests bit N in the given vector. -+static really_inline -+char testbit384(m384 val, unsigned int n) { ++static really_inline char testbit384(m384 val, unsigned int n) { + assert(n < sizeof(val) * 8); + m128 sub; + if (n < 128) { @@ -12418,280 +2291,151 @@ index 0000000..59ac642 + **** 512-bit Primitives + ****/ + -+#define eq512mask(a, b) _mm512_cmpeq_epi8_mask((a), (b)) -+#define masked_eq512mask(k, a, b) _mm512_mask_cmpeq_epi8_mask((k), (a), (b)) -+ -+static really_inline -+m512 zeroes512(void) { -+#if defined(HAVE_AVX512) -+ return _mm512_setzero_si512(); -+#else ++static really_inline m512 zeroes512(void) { + m512 rv = {zeroes256(), zeroes256()}; + return rv; -+#endif +} + -+static really_inline -+m512 ones512(void) { -+#if defined(HAVE_AVX512) -+ return _mm512_set1_epi8(0xFF); -+ //return _mm512_xor_si512(_mm512_setzero_si512(), _mm512_setzero_si512()); -+#else ++static really_inline m512 ones512(void) { + m512 rv = {ones256(), ones256()}; + return rv; -+#endif -+} -+ -+#if defined(HAVE_AVX512) -+static really_inline -+m512 set64x8(u8 a) { -+ return _mm512_set1_epi8(a); -+} -+ -+static really_inline -+m512 set8x64(u64a a) { -+ return _mm512_set1_epi64(a); -+} -+ -+static really_inline -+m512 set512_64(u64a hi_3, u64a hi_2, u64a hi_1, u64a hi_0, -+ u64a lo_3, u64a lo_2, u64a lo_1, u64a lo_0) { -+ return _mm512_set_epi64(hi_3, hi_2, hi_1, hi_0, -+ lo_3, lo_2, lo_1, lo_0); +} + -+static really_inline -+m512 swap256in512(m512 a) { -+ m512 idx = set512_64(3ULL, 2ULL, 1ULL, 0ULL, 7ULL, 6ULL, 5ULL, 4ULL); -+ return vpermq512(idx, a); -+} -+ -+static really_inline -+m512 set4x128(m128 a) { -+ return _mm512_broadcast_i32x4(a); -+} -+#endif -+ -+static really_inline -+m512 and512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return _mm512_and_si512(a, b); -+#else ++static really_inline m512 and512(m512 a, m512 b) { + m512 rv; + rv.lo = and256(a.lo, b.lo); + rv.hi = and256(a.hi, b.hi); + return rv; -+#endif +} + -+static really_inline -+m512 or512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return _mm512_or_si512(a, b); -+#else ++static really_inline m512 or512(m512 a, m512 b) { + m512 rv; + rv.lo = or256(a.lo, b.lo); + rv.hi = or256(a.hi, b.hi); + return rv; -+#endif +} + -+static really_inline -+m512 xor512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return _mm512_xor_si512(a, b); -+#else ++static really_inline m512 xor512(m512 a, m512 b) { + m512 rv; + rv.lo = xor256(a.lo, b.lo); + rv.hi = xor256(a.hi, b.hi); + return rv; -+#endif +} + -+static really_inline -+m512 not512(m512 a) { -+#if defined(HAVE_AVX512) -+ return _mm512_xor_si512(a, ones512()); -+#else ++static really_inline m512 not512(m512 a) { + m512 rv; + rv.lo = not256(a.lo); + rv.hi = not256(a.hi); + return rv; -+#endif +} + -+static really_inline -+m512 andnot512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return _mm512_andnot_si512(a, b); -+#else ++static really_inline m512 andnot512(m512 a, m512 b) { + m512 rv; + rv.lo = andnot256(a.lo, b.lo); + rv.hi = andnot256(a.hi, b.hi); + return rv; -+#endif +} + -+#if defined(HAVE_AVX512) -+static really_really_inline -+m512 lshift64_m512(m512 a, unsigned b) { -+#if defined(HAVE__BUILTIN_CONSTANT_P) -+ if (__builtin_constant_p(b)) { -+ return _mm512_slli_epi64(a, b); -+ } -+#endif -+ m128 x = _mm_cvtsi32_si128(b); -+ return _mm512_sll_epi64(a, x); -+} -+#else -+static really_really_inline -+m512 lshift64_m512(m512 a, unsigned b) { ++static really_really_inline m512 lshift64_m512(m512 a, unsigned b) { + m512 rv; + rv.lo = lshift64_m256(a.lo, b); + rv.hi = lshift64_m256(a.hi, b); + return rv; +} -+#endif -+ -+#if defined(HAVE_AVX512) -+#define rshift64_m512(a, b) _mm512_srli_epi64((a), (b)) -+#define rshift128_m512(a, count_immed) _mm512_bsrli_epi128(a, count_immed) -+#define lshift128_m512(a, count_immed) _mm512_bslli_epi128(a, count_immed) -+#endif + -+#if !defined(_MM_CMPINT_NE) -+#define _MM_CMPINT_NE 0x4 -+#endif -+ -+static really_inline -+int diff512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return !!_mm512_cmp_epi8_mask(a, b, _MM_CMPINT_NE); -+#else ++static really_inline int diff512(m512 a, m512 b) { + return diff256(a.lo, b.lo) || diff256(a.hi, b.hi); -+#endif +} + -+static really_inline -+int isnonzero512(m512 a) { -+#if defined(HAVE_AVX512) -+ return diff512(a, zeroes512()); -+#elif defined(HAVE_AVX2) -+ m256 x = or256(a.lo, a.hi); -+ return !!diff256(x, zeroes256()); -+#else ++static really_inline int isnonzero512(m512 a) { + m128 x = or128(a.lo.lo, a.lo.hi); + m128 y = or128(a.hi.lo, a.hi.hi); + return isnonzero128(or128(x, y)); -+#endif +} + +/** + * "Rich" version of diff512(). Takes two vectors a and b and returns a 16-bit + * mask indicating which 32-bit words contain differences. + */ -+static really_inline -+u32 diffrich512(m512 a, m512 b) { -+#if defined(HAVE_AVX512) -+ return _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_NE); -+#elif defined(HAVE_AVX2) -+ return diffrich256(a.lo, b.lo) | (diffrich256(a.hi, b.hi) << 8); -+#else -+ a.lo.lo = _mm_cmpeq_epi32(a.lo.lo, b.lo.lo); -+ a.lo.hi = _mm_cmpeq_epi32(a.lo.hi, b.lo.hi); -+ a.hi.lo = _mm_cmpeq_epi32(a.hi.lo, b.hi.lo); -+ a.hi.hi = _mm_cmpeq_epi32(a.hi.hi, b.hi.hi); -+ m128 packed = _mm_packs_epi16(_mm_packs_epi32(a.lo.lo, a.lo.hi), -+ _mm_packs_epi32(a.hi.lo, a.hi.hi)); -+ return ~(_mm_movemask_epi8(packed)) & 0xffff; -+#endif ++static really_inline u32 diffrich512(m512 a, m512 b) { ++ uint32x4_t x = vceqq_s32(a.lo.lo.vect_s32, b.lo.lo.vect_s32); ++ uint32x4_t y = vceqq_s32(a.lo.hi.vect_s32, b.lo.hi.vect_s32); ++ uint32x4_t z = vceqq_s32(a.hi.lo.vect_s32, b.hi.lo.vect_s32); ++ uint32x4_t w = vceqq_s32(a.hi.hi.vect_s32, b.hi.hi.vect_s32); ++ uint16x8_t p = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y)); ++ uint16x8_t q = vcombine_u16(vqmovn_u32(z), vqmovn_u32(w)); ++ ++ uint8x16_t input = vcombine_u8(vqmovn_u16(p), vqmovn_u16(q)); ++ ++ static const int8_t __attribute__((aligned(16))) ++ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; ++ uint8x8_t mask_and = vdup_n_u8(0x80); ++ int8x8_t mask_shift = vld1_s8(xr); ++ ++ uint8x8_t lo = vget_low_u8(input); ++ uint8x8_t hi = vget_high_u8(input); ++ ++ lo = vand_u8(lo, mask_and); ++ lo = vshl_u8(lo, mask_shift); ++ ++ hi = vand_u8(hi, mask_and); ++ hi = vshl_u8(hi, mask_shift); ++ ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); ++ lo = vpadd_u8(lo, lo); ++ ++ hi = vpadd_u8(hi, hi); ++ hi = vpadd_u8(hi, hi); ++ hi = vpadd_u8(hi, hi); ++ ++ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xffff; +} + +/** + * "Rich" version of diffrich(), 64-bit variant. Takes two vectors a and b and + * returns a 16-bit mask indicating which 64-bit words contain differences. + */ -+static really_inline -+u32 diffrich64_512(m512 a, m512 b) { -+ //TODO: cmp_epi64? ++static really_inline u32 diffrich64_512(m512 a, m512 b) { + u32 d = diffrich512(a, b); + return (d | (d >> 1)) & 0x55555555; +} + +// aligned load -+static really_inline -+m512 load512(const void *ptr) { -+#if defined(HAVE_AVX512) -+ return _mm512_load_si512(ptr); -+#else ++static really_inline m512 load512(const void *ptr) { + assert(ISALIGNED_N(ptr, alignof(m256))); -+ m512 rv = { load256(ptr), load256((const char *)ptr + 32) }; ++ m512 rv = {load256(ptr), load256((const char *)ptr + 32)}; + return rv; -+#endif +} + +// aligned store -+static really_inline -+void store512(void *ptr, m512 a) { ++static really_inline void store512(void *ptr, m512 a) { + assert(ISALIGNED_N(ptr, alignof(m512))); -+#if defined(HAVE_AVX512) -+ return _mm512_store_si512(ptr, a); -+#elif defined(HAVE_AVX2) -+ m512 *x = (m512 *)ptr; -+ store256(&x->lo, a.lo); -+ store256(&x->hi, a.hi); -+#else + ptr = assume_aligned(ptr, 16); + *(m512 *)ptr = a; -+#endif +} + +// unaligned load -+static really_inline -+m512 loadu512(const void *ptr) { -+#if defined(HAVE_AVX512) -+ return _mm512_loadu_si512(ptr); -+#else -+ m512 rv = { loadu256(ptr), loadu256((const char *)ptr + 32) }; ++static really_inline m512 loadu512(const void *ptr) { ++ m512 rv = {loadu256(ptr), loadu256((const char *)ptr + 32)}; + return rv; -+#endif -+} -+ -+#if defined(HAVE_AVX512) -+static really_inline -+m512 loadu_maskz_m512(__mmask64 k, const void *ptr) { -+ return _mm512_maskz_loadu_epi8(k, ptr); -+} -+ -+static really_inline -+m512 loadu_mask_m512(m512 src, __mmask64 k, const void *ptr) { -+ return _mm512_mask_loadu_epi8(src, k, ptr); -+} -+ -+static really_inline -+m512 set_mask_m512(__mmask64 k) { -+ return _mm512_movm_epi8(k); +} -+#endif + +// packed unaligned store of first N bytes -+static really_inline -+void storebytes512(void *ptr, m512 a, unsigned int n) { ++static really_inline void storebytes512(void *ptr, m512 a, unsigned int n) { + assert(n <= sizeof(a)); + memcpy(ptr, &a, n); +} + +// packed unaligned load of first N bytes, pad with zero -+static really_inline -+m512 loadbytes512(const void *ptr, unsigned int n) { ++static really_inline m512 loadbytes512(const void *ptr, unsigned int n) { + m512 a = zeroes512(); + assert(n <= sizeof(a)); + memcpy(&a, ptr, n); + return a; +} + -+static really_inline -+m512 mask1bit512(unsigned int n) { ++static really_inline m512 mask1bit512(unsigned int n) { + assert(n < sizeof(m512) * 8); + u32 mask_idx = ((n % 8) * 64) + 95; + mask_idx -= n / 8; @@ -12699,10 +2443,8 @@ index 0000000..59ac642 +} + +// switches on bit N in the given vector. -+static really_inline -+void setbit512(m512 *ptr, unsigned int n) { ++static really_inline void setbit512(m512 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); -+#if !defined(HAVE_AVX2) + m128 *sub; + if (n < 128) { + sub = &ptr->lo.lo; @@ -12714,25 +2456,11 @@ index 0000000..59ac642 + sub = &ptr->hi.hi; + } + setbit128(sub, n % 128); -+#elif defined(HAVE_AVX512) -+ *ptr = or512(mask1bit512(n), *ptr); -+#else -+ m256 *sub; -+ if (n < 256) { -+ sub = &ptr->lo; -+ } else { -+ sub = &ptr->hi; -+ n -= 256; -+ } -+ setbit256(sub, n); -+#endif +} + +// switches off bit N in the given vector. -+static really_inline -+void clearbit512(m512 *ptr, unsigned int n) { ++static really_inline void clearbit512(m512 *ptr, unsigned int n) { + assert(n < sizeof(*ptr) * 8); -+#if !defined(HAVE_AVX2) + m128 *sub; + if (n < 128) { + sub = &ptr->lo.lo; @@ -12744,25 +2472,11 @@ index 0000000..59ac642 + sub = &ptr->hi.hi; + } + clearbit128(sub, n % 128); -+#elif defined(HAVE_AVX512) -+ *ptr = andnot512(mask1bit512(n), *ptr); -+#else -+ m256 *sub; -+ if (n < 256) { -+ sub = &ptr->lo; -+ } else { -+ sub = &ptr->hi; -+ n -= 256; -+ } -+ clearbit256(sub, n); -+#endif +} + +// tests bit N in the given vector. -+static really_inline -+char testbit512(m512 val, unsigned int n) { ++static really_inline char testbit512(m512 val, unsigned int n) { + assert(n < sizeof(val) * 8); -+#if !defined(HAVE_AVX2) + m128 sub; + if (n < 128) { + sub = val.lo.lo; @@ -12774,131 +2488,76 @@ index 0000000..59ac642 + sub = val.hi.hi; + } + return testbit128(sub, n % 128); -+#elif defined(HAVE_AVX512) -+ const m512 mask = mask1bit512(n); -+ return !!_mm512_test_epi8_mask(mask, val); -+#else -+ m256 sub; -+ if (n < 256) { -+ sub = val.lo; -+ } else { -+ sub = val.hi; -+ n -= 256; -+ } -+ return testbit256(sub, n); -+#endif +} ++#pragma GCC diagnostic pop + +#endif -diff --git a/src/util/state_compress.c b/src/util/state_compress.c -index 7238849..4422403 100644 ---- a/src/util/state_compress.c -+++ b/src/util/state_compress.c -@@ -150,7 +150,7 @@ m128 loadcompressed128_32bit(const void *ptr, m128 mvec) { - u32 x[4] = { expand32(v[0], m[0]), expand32(v[1], m[1]), - expand32(v[2], m[2]), expand32(v[3], m[3]) }; - -- return _mm_set_epi32(x[3], x[2], x[1], x[0]); -+ return set32x4(x[3], x[2], x[1], x[0]); - } - #endif - -@@ -158,7 +158,7 @@ m128 loadcompressed128_32bit(const void *ptr, m128 mvec) { - static really_inline - m128 loadcompressed128_64bit(const void *ptr, m128 mvec) { - // First, decompose our vectors into 64-bit chunks. -- u64a m[2] = { movq(mvec), movq(_mm_srli_si128(mvec, 8)) }; -+ u64a m[2] = { movq(mvec), movq(rshiftbyte_m128(mvec, 8)) }; - - u32 bits[2] = { popcount64(m[0]), popcount64(m[1]) }; - u64a v[2]; -@@ -167,7 +167,7 @@ m128 loadcompressed128_64bit(const void *ptr, m128 mvec) { - - u64a x[2] = { expand64(v[0], m[0]), expand64(v[1], m[1]) }; - -- return _mm_set_epi64x(x[1], x[0]); -+ return set64x2(x[1], x[0]); - } - #endif - -@@ -264,8 +264,8 @@ m256 loadcompressed256_32bit(const void *ptr, m256 mvec) { - expand32(v[6], m[6]), expand32(v[7], m[7]) }; - - #if !defined(HAVE_AVX2) -- m256 xvec = { .lo = _mm_set_epi32(x[3], x[2], x[1], x[0]), -- .hi = _mm_set_epi32(x[7], x[6], x[5], x[4]) }; -+ m256 xvec = { .lo = set32x4(x[3], x[2], x[1], x[0]), -+ .hi = set32x4(x[7], x[6], x[5], x[4]) }; - #else - m256 xvec = _mm256_set_epi32(x[7], x[6], x[5], x[4], - x[3], x[2], x[1], x[0]); -@@ -291,8 +291,8 @@ m256 loadcompressed256_64bit(const void *ptr, m256 mvec) { - expand64(v[2], m[2]), expand64(v[3], m[3]) }; +diff --git a/src/util/simd_types.h b/src/util/simd_types.h +index 962cad6..b3f96ea 100644 +--- a/src/util/simd_types.h ++++ b/src/util/simd_types.h +@@ -35,6 +35,23 @@ + #include "ue2common.h" - #if !defined(HAVE_AVX2) -- m256 xvec = { .lo = _mm_set_epi64x(x[1], x[0]), -- .hi = _mm_set_epi64x(x[3], x[2]) }; -+ m256 xvec = { .lo = set64x2(x[1], x[0]), -+ .hi = set64x2(x[3], x[2]) }; + #if defined(HAVE_SSE2) ++typedef __m128i m128; ++#elif defined(HAVE_NEON) ++#include "arm_neon.h" ++ ++typedef union { ++ int8x16_t vect_s8; ++ int16x8_t vect_s16; ++ int32x4_t vect_s32; ++ int64x2_t vect_s64; ++ uint8x16_t vect_u8; ++ uint16x8_t vect_u16; ++ uint32x4_t vect_u32; ++ uint64x2_t vect_u64; ++} __m128i; ++typedef float32x4_t __m128; ++typedef float64x2_t __m128d; ++ + typedef __m128i m128; #else - m256 xvec = _mm256_set_epi64x(x[3], x[2], x[1], x[0]); + typedef struct ALIGN_DIRECTIVE {u64a hi; u64a lo;} m128; +diff --git a/src/util/simd_utils.h b/src/util/simd_utils.h +new file mode 100644 +index 0000000..9588d97 +--- /dev/null ++++ b/src/util/simd_utils.h +@@ -0,0 +1,13 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. ++ ++#ifndef SIMD_UTILS ++#define SIMD_UTILS ++ ++#if defined(__x86_64__) ++#include "simd_x86.h" ++#elif defined(__aarch64__) ++#include "simd_arm.h" ++#endif ++ ++#endif +diff --git a/src/util/simd_x86.h b/src/util/simd_x86.h +index 5fa727e..5daaa74 100644 +--- a/src/util/simd_x86.h ++++ b/src/util/simd_x86.h +@@ -1417,4 +1417,14 @@ char testbit512(m512 val, unsigned int n) { #endif -@@ -402,9 +402,9 @@ m384 loadcompressed384_32bit(const void *ptr, m384 mvec) { - expand32(v[8], m[8]), expand32(v[9], m[9]), - expand32(v[10], m[10]), expand32(v[11], m[11]) }; - -- m384 xvec = { .lo = _mm_set_epi32(x[3], x[2], x[1], x[0]), -- .mid = _mm_set_epi32(x[7], x[6], x[5], x[4]), -- .hi = _mm_set_epi32(x[11], x[10], x[9], x[8]) }; -+ m384 xvec = { .lo = set32x4(x[3], x[2], x[1], x[0]), -+ .mid = set32x4(x[7], x[6], x[5], x[4]), -+ .hi = set32x4(x[11], x[10], x[9], x[8]) }; - return xvec; } - #endif -@@ -427,9 +427,9 @@ m384 loadcompressed384_64bit(const void *ptr, m384 mvec) { - expand64(v[2], m[2]), expand64(v[3], m[3]), - expand64(v[4], m[4]), expand64(v[5], m[5]) }; -- m384 xvec = { .lo = _mm_set_epi64x(x[1], x[0]), -- .mid = _mm_set_epi64x(x[3], x[2]), -- .hi = _mm_set_epi64x(x[5], x[4]) }; -+ m384 xvec = { .lo = set64x2(x[1], x[0]), -+ .mid = set64x2(x[3], x[2]), -+ .hi = set64x2(x[5], x[4]) }; - return xvec; - } - #endif -@@ -558,10 +558,10 @@ m512 loadcompressed512_32bit(const void *ptr, m512 mvec) { - xvec.hi = _mm256_set_epi32(x[15], x[14], x[13], x[12], - x[11], x[10], x[9], x[8]); - #else -- xvec.lo.lo = _mm_set_epi32(x[3], x[2], x[1], x[0]); -- xvec.lo.hi = _mm_set_epi32(x[7], x[6], x[5], x[4]); -- xvec.hi.lo = _mm_set_epi32(x[11], x[10], x[9], x[8]); -- xvec.hi.hi = _mm_set_epi32(x[15], x[14], x[13], x[12]); -+ xvec.lo.lo = set32x4(x[3], x[2], x[1], x[0]); -+ xvec.lo.hi = set32x4(x[7], x[6], x[5], x[4]); -+ xvec.hi.lo = set32x4(x[11], x[10], x[9], x[8]); -+ xvec.hi.hi = set32x4(x[15], x[14], x[13], x[12]); - #endif - return xvec; - } -@@ -594,10 +594,10 @@ m512 loadcompressed512_64bit(const void *ptr, m512 mvec) { - m512 xvec = { .lo = _mm256_set_epi64x(x[3], x[2], x[1], x[0]), - .hi = _mm256_set_epi64x(x[7], x[6], x[5], x[4])}; - #else -- m512 xvec = { .lo = { _mm_set_epi64x(x[1], x[0]), -- _mm_set_epi64x(x[3], x[2]) }, -- .hi = { _mm_set_epi64x(x[5], x[4]), -- _mm_set_epi64x(x[7], x[6]) } }; -+ m512 xvec = { .lo = { set64x2(x[1], x[0]), -+ set64x2(x[3], x[2]) }, -+ .hi = { set64x2(x[5], x[4]), -+ set64x2(x[7], x[6]) } }; ++static really_inline m128 set2x64(u64a c) ++{ ++ return _mm_set1_epi32(c); ++} ++ ++static really_inline m128 set32x4(int i3, int i2, int i1, int i0) ++{ ++ return _mm_set_epi32(i3, i2, i1, i0); ++} ++ #endif - return xvec; - } diff --git a/tools/hscollider/CMakeLists.txt b/tools/hscollider/CMakeLists.txt index a4d71b2..0c41ab9 100644 --- a/tools/hscollider/CMakeLists.txt @@ -13400,59 +3059,10 @@ index 0000000..5391473 + return (cs != FileCorporaParser_error) && (p == pe); +} diff --git a/unit/internal/simd_utils.cpp b/unit/internal/simd_utils.cpp -index 623c2c9..d6d52a2 100644 +index 623c2c9..22945d6 100644 --- a/unit/internal/simd_utils.cpp +++ b/unit/internal/simd_utils.cpp -@@ -40,7 +40,7 @@ using namespace ue2; - namespace { - - // Switch one bit on in a bitmask. --template -+template - Mask setbit(unsigned int bit) { - union { - Mask simd; -@@ -148,7 +148,7 @@ m256 simd_lshift64(const m256 &a, unsigned i) { return lshift64_m256(a, i); } - m384 simd_lshift64(const m384 &a, unsigned i) { return lshift64_m384(a, i); } - m512 simd_lshift64(const m512 &a, unsigned i) { return lshift64_m512(a, i); } - --template -+template - class SimdUtilsTest : public testing::Test { - // empty - }; -@@ -260,9 +260,9 @@ TYPED_TEST(SimdUtilsTest, or2) { - - for (unsigned j = 0; j < 8; j++) { - for (unsigned i = 0; i < 32; i++) { -- m256 x = setbit(j*32+i); -+ m256 x = setbit(j * 32 + i); - m256 y = zeroes256(); -- ASSERT_EQ(1U << j, diffrich256(x, y)) << "bit " << j*32+i << " not happy"; -+ ASSERT_EQ(1U << j, diffrich256(x, y)) << "bit " << j * 32 + i << " not happy"; - } - } - -@@ -431,8 +431,8 @@ TYPED_TEST(SimdUtilsTest, testbit) { - for (unsigned int i = 0; i < total_bits; i++) { - TypeParam a = setbit(i); - for (unsigned int j = 0; j < total_bits; j++) { -- ASSERT_EQ(i == j ? 1 : 0, simd_testbit(a, j)) << "bit " << i -- << " is wrong"; -+ ASSERT_EQ(i == j ? 1 : 0, simd_testbit(a, j)) -+ << "bit " << i << " is wrong"; - } - } - } -@@ -455,7 +455,6 @@ TYPED_TEST(SimdUtilsTest, setbit) { - simd_setbit(&a, i); - } - ASSERT_FALSE(simd_diff(simd_ones(), a)); -- - } - - TYPED_TEST(SimdUtilsTest, diffrich) { -@@ -663,12 +662,11 @@ TEST(SimdUtilsTest, movq) { +@@ -663,7 +663,7 @@ TEST(SimdUtilsTest, movq) { ASSERT_EQ(0, memcmp(cmp, &simd, sizeof(simd))); ASSERT_EQ(0, memcmp(cmp, &r, sizeof(r))); @@ -13461,152 +3071,6 @@ index 623c2c9..d6d52a2 100644 r = movq(simd); ASSERT_EQ(r, 0x123456789abcdef); } - -- - TEST(SimdUtilsTest, set16x8) { - char cmp[sizeof(m128)]; - -@@ -680,7 +678,7 @@ TEST(SimdUtilsTest, set16x8) { - } - - TEST(SimdUtilsTest, set4x32) { -- u32 cmp[4] = { 0x12345678, 0x12345678, 0x12345678, 0x12345678 }; -+ u32 cmp[4] = {0x12345678, 0x12345678, 0x12345678, 0x12345678}; - m128 simd = set4x32(cmp[0]); - ASSERT_EQ(0, memcmp(cmp, &simd, sizeof(simd))); - } -@@ -714,51 +712,51 @@ TEST(SimdUtilsTest, variableByteShift128) { - char base[] = "0123456789ABCDEF"; - m128 in = loadu128(base); - -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 0), -- variable_byte_shift_m128(in, 0))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 1), -- variable_byte_shift_m128(in, -1))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 2), -- variable_byte_shift_m128(in, -2))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 3), -- variable_byte_shift_m128(in, -3))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 4), -- variable_byte_shift_m128(in, -4))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 5), -- variable_byte_shift_m128(in, -5))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 6), -- variable_byte_shift_m128(in, -6))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 7), -- variable_byte_shift_m128(in, -7))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 8), -- variable_byte_shift_m128(in, -8))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 9), -- variable_byte_shift_m128(in, -9))); -- EXPECT_TRUE(!diff128(rshiftbyte_m128(in, 10), -- variable_byte_shift_m128(in, -10))); -- -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 0), -- variable_byte_shift_m128(in, 0))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 1), -- variable_byte_shift_m128(in, 1))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 2), -- variable_byte_shift_m128(in, 2))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 3), -- variable_byte_shift_m128(in, 3))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 4), -- variable_byte_shift_m128(in, 4))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 5), -- variable_byte_shift_m128(in, 5))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 6), -- variable_byte_shift_m128(in, 6))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 7), -- variable_byte_shift_m128(in, 7))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 8), -- variable_byte_shift_m128(in, 8))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 9), -- variable_byte_shift_m128(in, 9))); -- EXPECT_TRUE(!diff128(lshiftbyte_m128(in, 10), -- variable_byte_shift_m128(in, 10))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 0), variable_byte_shift_m128(in, 0))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 1), variable_byte_shift_m128(in, -1))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 2), variable_byte_shift_m128(in, -2))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 3), variable_byte_shift_m128(in, -3))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 4), variable_byte_shift_m128(in, -4))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 5), variable_byte_shift_m128(in, -5))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 6), variable_byte_shift_m128(in, -6))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 7), variable_byte_shift_m128(in, -7))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 8), variable_byte_shift_m128(in, -8))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 9), variable_byte_shift_m128(in, -9))); -+ EXPECT_TRUE( -+ !diff128(rshiftbyte_m128(in, 10), variable_byte_shift_m128(in, -10))); -+ -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 0), variable_byte_shift_m128(in, 0))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 1), variable_byte_shift_m128(in, 1))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 2), variable_byte_shift_m128(in, 2))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 3), variable_byte_shift_m128(in, 3))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 4), variable_byte_shift_m128(in, 4))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 5), variable_byte_shift_m128(in, 5))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 6), variable_byte_shift_m128(in, 6))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 7), variable_byte_shift_m128(in, 7))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 8), variable_byte_shift_m128(in, 8))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 9), variable_byte_shift_m128(in, 9))); -+ EXPECT_TRUE( -+ !diff128(lshiftbyte_m128(in, 10), variable_byte_shift_m128(in, 10))); - - EXPECT_TRUE(!diff128(zeroes128(), variable_byte_shift_m128(in, 16))); - EXPECT_TRUE(!diff128(zeroes128(), variable_byte_shift_m128(in, -16))); -@@ -785,12 +783,12 @@ TEST(SimdUtilsTest, min_u8_m128) { - } - - TEST(SimdUtilsTest, sadd_u8_m128) { -- unsigned char base1[] = {0, 0x80, 0xff, 'A', '1', '2', '3', '4', -- '1', '2', '3', '4', '1', '2', '3', '4'}; -- unsigned char base2[] = {'a', 0x80, 'b', 'A', 0x10, 0x10, 0x10, 0x10, -- 0x30, 0x30, 0x30, 0x30, 0, 0, 0, 0}; -+ unsigned char base1[] = {0, 0x80, 0xff, 'A', '1', '2', '3', '4', -+ '1', '2', '3', '4', '1', '2', '3', '4'}; -+ unsigned char base2[] = {'a', 0x80, 'b', 'A', 0x10, 0x10, 0x10, 0x10, -+ 0x30, 0x30, 0x30, 0x30, 0, 0, 0, 0}; - unsigned char expec[] = {'a', 0xff, 0xff, 0x82, 'A', 'B', 'C', 'D', -- 'a', 'b', 'c', 'd', '1', '2', '3', '4'}; -+ 'a', 'b', 'c', 'd', '1', '2', '3', '4'}; - m128 in1 = loadu128(base1); - m128 in2 = loadu128(base2); - m128 result = sadd_u8_m128(in1, in2); -@@ -799,11 +797,11 @@ TEST(SimdUtilsTest, sadd_u8_m128) { - - TEST(SimdUtilsTest, sub_u8_m128) { - unsigned char base1[] = {'a', 0xff, 0xff, 0x82, 'A', 'B', 'C', 'D', -- 'a', 'b', 'c', 'd', '1', '2', '3', '4'}; -- unsigned char base2[] = {0, 0x80, 0xff, 'A', '1', '2', '3', '4', -- '1', '2', '3', '4', '1', '2', '3', '4'}; -- unsigned char expec[] = {'a', 0x7f, 0, 'A', 0x10, 0x10, 0x10, 0x10, -- 0x30, 0x30, 0x30, 0x30, 0, 0, 0, 0}; -+ 'a', 'b', 'c', 'd', '1', '2', '3', '4'}; -+ unsigned char base2[] = {0, 0x80, 0xff, 'A', '1', '2', '3', '4', -+ '1', '2', '3', '4', '1', '2', '3', '4'}; -+ unsigned char expec[] = {'a', 0x7f, 0, 'A', 0x10, 0x10, 0x10, 0x10, -+ 0x30, 0x30, 0x30, 0x30, 0, 0, 0, 0}; - m128 in1 = loadu128(base1); - m128 in2 = loadu128(base2); - m128 result = sub_u8_m128(in1, in2); diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt index ea942ef..d7bef50 100644 --- a/util/CMakeLists.txt @@ -14030,5 +3494,5 @@ index 0000000..687fc39 + return (cs != ExpressionParser_error) && (p == pe); +} -- -2.31.1 +2.39.0 diff --git a/hyperscan.spec b/hyperscan.spec index 96240118b16c233eb54a2e8fa728753677b14752..114cf1cd70222c4b7f8a39c22cedee1c4dedc0db 100644 --- a/hyperscan.spec +++ b/hyperscan.spec @@ -1,6 +1,6 @@ Name: hyperscan -Version: 5.4.0 -Release: 3 +Version: 5.4.2 +Release: 1 Summary: High-performance regular expression matching library License: BSD @@ -8,11 +8,7 @@ URL: https://www.hyperscan.io/ Source0: https://github.com/intel/%{name}/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz Patch0: hyperscan-aarch64-support.patch -Patch1: Fix-build-error-on-x86_64.patch -Patch2: Fix-hyperscan-gcc10.patch -# https://github.com/intel/hyperscan/commit/7d644e7ba27eaadda753febf0b142faa9affbbca -Patch3: backport-Fix-segfaults-on-allocation-failure.patch -Patch4: CVE-2022-29486.patch +Patch1: Fix-hyperscan-gcc10.patch BuildRequires: gcc-c++ BuildRequires: boost-devel @@ -55,7 +51,15 @@ This package provides the libraries, include files and other resources needed for developing Hyperscan applications. %prep -%autosetup -n %{name}-%{version} -p1 +%setup -q -n %{name}-%{version} +cd %{_builddir}/%{name}-%{version} +mv src/util/simd_utils.h src/util/simd_x86.h +sed -i 's/SIMD_UTILS/SIMD_X86/' src/util/simd_x86.h +sed -i 's/_mm_set_epi32/set32x4/' src/util/state_compress.c +sed -i 's/_mm_set_epi64x/set64x2/' src/util/state_compress.c +sed -i 's/_mm_srli_si128/rshiftbyte_m128/' src/util/state_compress.c +cd - +%autopatch -p1 %build %cmake -DBUILD_SHARED_LIBS:BOOL=ON -DBUILD_STATIC_AND_SHARED:BOOL=OFF . @@ -81,6 +85,9 @@ needed for developing Hyperscan applications. %{_includedir}/hs/ %changelog +* Fri Sep 01 2023 wangkai <13474090681@163.com> - 5.4.2-1 +- Update to 5.4.2 for fix CVE-2023-28711 + * Thu Apr 20 2023 wangkai <13474090681@163.com> - 5.4.0-3 - Fix CVE-2022-29486