15 #ifndef RAPIDJSON_INTERNAL_REGEX_H_
16 #define RAPIDJSON_INTERNAL_REGEX_H_
18 #include "../allocators.h"
19 #include "../stream.h"
24 RAPIDJSON_DIAG_OFF(padded)
25 RAPIDJSON_DIAG_OFF(
switch -
enum)
26 #elif defined(_MSC_VER)
28 RAPIDJSON_DIAG_OFF(4512)
33 RAPIDJSON_DIAG_OFF(effc++)
36 #ifndef RAPIDJSON_REGEX_VERBOSE
37 #define RAPIDJSON_REGEX_VERBOSE 0
46 template <
typename SourceStream,
typename Encoding>
51 unsigned Peek() {
return codepoint_; }
54 unsigned c = codepoint_;
63 if(!Encoding::Decode(ss_, &codepoint_))
74 static const SizeType kRegexInvalidState =
78 template <
typename Encoding,
typename Allocator>
79 class GenericRegexSearch;
113 template <
typename Encoding,
typename Allocator = CrtAllocator>
118 typedef typename Encoding::Ch
Ch;
119 template <
typename,
typename>
124 allocator_(allocator ? allocator : ownAllocator_),
125 states_(allocator_, 256),
126 ranges_(allocator_, 256),
127 root_(kRegexInvalidState),
140 bool IsValid()
const {
return root_ != kRegexInvalidState; }
153 static const unsigned kAnyCharacterClass = 0xFFFFFFFF;
154 static const unsigned kRangeCharacterClass = 0xFFFFFFFE;
155 static const unsigned kRangeNegationFlag = 0x80000000;
183 return states_.template Bottom<State>()[index];
186 const State& GetState(
SizeType index)
const
189 return states_.template Bottom<State>()[index];
195 return ranges_.template Bottom<Range>()[index];
198 const Range& GetRange(
SizeType index)
const
201 return ranges_.template Bottom<Range>()[index];
204 template <
typename InputStream>
205 void Parse(DecodedStream<InputStream, Encoding>& ds)
207 Stack<Allocator> operandStack(allocator_, 256);
208 Stack<Allocator> operatorStack(allocator_, 256);
209 Stack<Allocator> atomCountStack(allocator_, 256);
211 *atomCountStack.template Push<unsigned>() = 0;
214 while(ds.Peek() != 0)
216 switch(codepoint = ds.Take())
218 case '^': anchorBegin_ =
true;
break;
220 case '$': anchorEnd_ =
true;
break;
223 while(!operatorStack.Empty() &&
224 *operatorStack.template Top<Operator>() < kAlternation)
225 if(!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
227 *operatorStack.template Push<Operator>() = kAlternation;
228 *atomCountStack.template Top<unsigned>() = 0;
232 *operatorStack.template Push<Operator>() = kLeftParenthesis;
233 *atomCountStack.template Push<unsigned>() = 0;
237 while(!operatorStack.Empty() &&
238 *operatorStack.template Top<Operator>() != kLeftParenthesis)
239 if(!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
241 if(operatorStack.Empty())
243 operatorStack.template Pop<Operator>(1);
244 atomCountStack.template Pop<unsigned>(1);
245 ImplicitConcatenation(atomCountStack, operatorStack);
249 if(!Eval(operandStack, kZeroOrOne))
254 if(!Eval(operandStack, kZeroOrMore))
259 if(!Eval(operandStack, kOneOrMore))
265 if(!ParseUnsigned(ds, &n))
272 m = kInfinityQuantifier;
273 else if(!ParseUnsigned(ds, &m) || m < n)
279 if(!EvalQuantifier(operandStack, n, m) || ds.Peek() !=
'}')
286 PushOperand(operandStack, kAnyCharacterClass);
287 ImplicitConcatenation(atomCountStack, operatorStack);
292 if(!ParseRange(ds, &range))
294 SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass);
295 GetState(s).rangeStart = range;
296 *operandStack.template Push<Frag>() = Frag(s, s, s);
298 ImplicitConcatenation(atomCountStack, operatorStack);
302 if(!CharacterEscape(ds, &codepoint))
305 RAPIDJSON_DELIBERATE_FALLTHROUGH;
308 PushOperand(operandStack, codepoint);
309 ImplicitConcatenation(atomCountStack, operatorStack);
313 while(!operatorStack.Empty())
314 if(!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
318 if(operandStack.GetSize() ==
sizeof(Frag))
320 Frag* e = operandStack.template Pop<Frag>(1);
321 Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0));
324 #if RAPIDJSON_REGEX_VERBOSE
325 printf(
"root: %d\n", root_);
326 for(
SizeType i = 0; i < stateCount_; i++)
328 State& s = GetState(i);
329 printf(
"[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (
char)s.codepoint);
338 State* s = states_.template Push<State>();
341 s->codepoint = codepoint;
342 s->rangeStart = kRegexInvalidRange;
343 return stateCount_++;
346 void PushOperand(Stack<Allocator>& operandStack,
unsigned codepoint)
348 SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint);
349 *operandStack.template Push<Frag>() = Frag(s, s, s);
352 void ImplicitConcatenation(Stack<Allocator>& atomCountStack, Stack<Allocator>& operatorStack)
354 if(*atomCountStack.template Top<unsigned>())
355 *operatorStack.template Push<Operator>() = kConcatenation;
356 (*atomCountStack.template Top<unsigned>())++;
362 while(GetState(l1).out != kRegexInvalidState)
363 l1 = GetState(l1).out;
364 GetState(l1).out = l2;
370 for(
SizeType next; l != kRegexInvalidState; l = next)
372 next = GetState(l).out;
377 bool Eval(Stack<Allocator>& operandStack, Operator op)
384 Frag e2 = *operandStack.template Pop<Frag>(1);
385 Frag e1 = *operandStack.template Pop<Frag>(1);
386 Patch(e1.out, e2.start);
387 *operandStack.template Push<Frag>() =
388 Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex));
393 if(operandStack.GetSize() >=
sizeof(Frag) * 2)
395 Frag e2 = *operandStack.template Pop<Frag>(1);
396 Frag e1 = *operandStack.template Pop<Frag>(1);
397 SizeType s = NewState(e1.start, e2.start, 0);
398 *operandStack.template Push<Frag>() =
399 Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex));
405 if(operandStack.GetSize() >=
sizeof(Frag))
407 Frag e = *operandStack.template Pop<Frag>(1);
408 SizeType s = NewState(kRegexInvalidState, e.start, 0);
409 *operandStack.template Push<Frag>() = Frag(s, Append(e.out, s), e.minIndex);
415 if(operandStack.GetSize() >=
sizeof(Frag))
417 Frag e = *operandStack.template Pop<Frag>(1);
418 SizeType s = NewState(kRegexInvalidState, e.start, 0);
420 *operandStack.template Push<Frag>() = Frag(s, s, e.minIndex);
426 if(operandStack.GetSize() >=
sizeof(Frag))
428 Frag e = *operandStack.template Pop<Frag>(1);
429 SizeType s = NewState(kRegexInvalidState, e.start, 0);
431 *operandStack.template Push<Frag>() = Frag(e.start, s, e.minIndex);
442 bool EvalQuantifier(Stack<Allocator>& operandStack,
unsigned n,
unsigned m)
451 else if(m == kInfinityQuantifier)
452 Eval(operandStack, kZeroOrMore);
455 Eval(operandStack, kZeroOrOne);
456 for(
unsigned i = 0; i < m - 1; i++)
457 CloneTopOperand(operandStack);
458 for(
unsigned i = 0; i < m - 1; i++)
459 Eval(operandStack, kConcatenation);
464 for(
unsigned i = 0; i < n - 1; i++)
465 CloneTopOperand(operandStack);
467 if(m == kInfinityQuantifier)
468 Eval(operandStack, kOneOrMore);
471 CloneTopOperand(operandStack);
472 Eval(operandStack, kZeroOrOne);
473 for(
unsigned i = n; i < m - 1; i++)
474 CloneTopOperand(operandStack);
475 for(
unsigned i = n; i < m; i++)
476 Eval(operandStack, kConcatenation);
479 for(
unsigned i = 0; i < n - 1; i++)
480 Eval(operandStack, kConcatenation);
487 void CloneTopOperand(Stack<Allocator>& operandStack)
490 *operandStack.template Top<Frag>();
494 State* s = states_.template Push<State>(count);
495 memcpy(s, &GetState(src.minIndex), count *
sizeof(State));
498 if(s[j].out != kRegexInvalidState)
500 if(s[j].out1 != kRegexInvalidState)
503 *operandStack.template Push<Frag>() =
504 Frag(src.start + count, src.out + count, src.minIndex + count);
505 stateCount_ += count;
508 template <
typename InputStream>
509 bool ParseUnsigned(DecodedStream<InputStream, Encoding>& ds,
unsigned* u)
512 if(ds.Peek() <
'0' || ds.Peek() >
'9')
514 while(ds.Peek() >=
'0' && ds.Peek() <=
'9')
516 if(r >= 429496729 && ds.Peek() >
'5')
518 r = r * 10 + (ds.Take() -
'0');
524 template <
typename InputStream>
525 bool ParseRange(DecodedStream<InputStream, Encoding>& ds,
SizeType* range)
530 SizeType start = kRegexInvalidRange;
531 SizeType current = kRegexInvalidRange;
533 while((codepoint = ds.Take()) != 0)
548 if(start == kRegexInvalidRange)
554 GetRange(current).next = r;
557 GetRange(start).start |= kRangeNegationFlag;
567 else if(!CharacterEscape(ds, &codepoint))
570 RAPIDJSON_DELIBERATE_FALLTHROUGH;
582 RAPIDJSON_DELIBERATE_FALLTHROUGH;
586 if(current != kRegexInvalidRange)
587 GetRange(current).next = r;
588 if(start == kRegexInvalidRange)
597 GetRange(current).end = codepoint;
605 SizeType NewRange(
unsigned codepoint)
607 Range* r = ranges_.template Push<Range>();
608 r->start = r->end = codepoint;
609 r->next = kRegexInvalidRange;
610 return rangeCount_++;
613 template <
typename InputStream>
614 bool CharacterEscape(DecodedStream<InputStream, Encoding>& ds,
unsigned* escapedCodepoint)
617 switch(codepoint = ds.Take())
632 case '\\': *escapedCodepoint = codepoint;
return true;
633 case 'f': *escapedCodepoint = 0x000C;
return true;
634 case 'n': *escapedCodepoint = 0x000A;
return true;
635 case 'r': *escapedCodepoint = 0x000D;
return true;
636 case 't': *escapedCodepoint = 0x0009;
return true;
637 case 'v': *escapedCodepoint = 0x000B;
return true;
638 default:
return false;
644 Stack<Allocator> states_;
645 Stack<Allocator> ranges_;
650 static const unsigned kInfinityQuantifier = ~0u;
657 template <
typename RegexType,
typename Allocator = CrtAllocator>
662 typedef typename Encoding::Ch
Ch;
666 allocator_(allocator),
668 state0_(allocator, 0),
669 state1_(allocator, 0),
675 stateSet_ =
static_cast<uint32_t*
>(allocator_->Malloc(GetStateSetSize()));
676 state0_.template Reserve<SizeType>(regex_.stateCount_);
677 state1_.template Reserve<SizeType>(regex_.stateCount_);
686 template <
typename InputStream>
689 return SearchWithAnchoring(is,
true,
true);
698 template <
typename InputStream>
701 return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_);
711 typedef typename RegexType::State State;
712 typedef typename RegexType::Range Range;
714 template <
typename InputStream>
715 bool SearchWithAnchoring(InputStream& is,
bool anchorBegin,
bool anchorEnd)
721 const size_t stateSetSize = GetStateSetSize();
722 std::memset(stateSet_, 0, stateSetSize);
724 bool matched = AddState(*current, regex_.root_);
726 while(!current->
Empty() && (codepoint = ds.Take()) != 0)
728 std::memset(stateSet_, 0, stateSetSize);
731 for(
const SizeType* s = current->template Bottom<SizeType>();
732 s != current->template End<SizeType>();
735 const State& sr = regex_.GetState(*s);
736 if(sr.codepoint == codepoint || sr.codepoint == RegexType::kAnyCharacterClass ||
737 (sr.codepoint == RegexType::kRangeCharacterClass &&
738 MatchRange(sr.rangeStart, codepoint)))
740 matched = AddState(*next, sr.out) || matched;
741 if(!anchorEnd && matched)
745 AddState(*next, regex_.root_);
753 size_t GetStateSetSize()
const {
return (regex_.stateCount_ + 31) / 32 * 4; }
756 bool AddState(Stack<Allocator>& l,
SizeType index)
760 const State& s = regex_.GetState(index);
761 if(s.out1 != kRegexInvalidState)
763 bool matched = AddState(l, s.out);
764 return AddState(l, s.out1) || matched;
766 else if(!(stateSet_[index >> 5] & (1u << (index & 31))))
768 stateSet_[index >> 5] |= (1u << (index & 31));
769 *l.template PushUnsafe<SizeType>() = index;
771 return s.out == kRegexInvalidState;
775 bool MatchRange(
SizeType rangeIndex,
unsigned codepoint)
const
777 bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0;
778 while(rangeIndex != kRegexInvalidRange)
780 const Range& r = regex_.GetRange(rangeIndex);
781 if(codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end)
788 const RegexType& regex_;
791 Stack<Allocator> state0_;
792 Stack<Allocator> state1_;
806 #if defined(__clang__) || defined(_MSC_VER)
void Free(A &a, T *p, size_t n=1)
Definition: allocators.h:485
DecodedStream(SourceStream &ss)
Definition: regex.h:50
unsigned Take()
Definition: regex.h:52
unsigned Peek()
Definition: regex.h:51
Regular expression engine with subset of ECMAscript grammar.
Definition: regex.h:115
bool IsValid() const
Definition: regex.h:140
GenericRegex(const Ch *source, Allocator *allocator=0)
Definition: regex.h:122
Encoding::Ch Ch
Definition: regex.h:118
~GenericRegex()
Definition: regex.h:138
Encoding EncodingType
Definition: regex.h:117
GenericRegexSearch(const RegexType ®ex, Allocator *allocator=0)
Definition: regex.h:664
bool Search(InputStream &is)
Definition: regex.h:699
RegexType::EncodingType Encoding
Definition: regex.h:661
Encoding::Ch Ch
Definition: regex.h:662
bool Search(const Ch *s)
Definition: regex.h:704
bool Match(const Ch *s)
Definition: regex.h:692
~GenericRegexSearch()
Definition: regex.h:680
bool Match(InputStream &is)
Definition: regex.h:687
A type-unsafe stack for storing different types of data.
Definition: stack.h:38
bool Empty() const
Definition: stack.h:205
Concept for allocating, resizing and freeing memory block.
Concept for encoding of Unicode characters.
#define RAPIDJSON_ASSERT(x)
Assertion.
Definition: rapidjson.h:451
#define RAPIDJSON_NAMESPACE_BEGIN
provide custom rapidjson namespace (opening expression)
Definition: rapidjson.h:121
#define RAPIDJSON_NAMESPACE_END
provide custom rapidjson namespace (closing expression)
Definition: rapidjson.h:124
Definition: allocators.h:459
GenericRegexSearch< Regex > RegexSearch
Definition: regex.h:797
void Swap(T &a, T &b) RAPIDJSON_NOEXCEPT
Custom swap() to avoid dependency on C++ <algorithm> header.
Definition: swap.h:33
GenericRegex< UTF8<> > Regex
Definition: regex.h:796
const CharType(& source)[N]
Definition: pointer.h:1559
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Definition: pointer.h:1517
#define RAPIDJSON_DELETE(x)
! customization point for global delete
Definition: rapidjson.h:746
RAPIDJSON_NAMESPACE_BEGIN typedef unsigned SizeType
Size type (for string lengths, array sizes, etc.)
Definition: rapidjson.h:429
#define RAPIDJSON_NEW(TypeName)
! customization point for global new
Definition: rapidjson.h:742
unsigned int uint32_t
Definition: stdint.h:126
Read-only string stream.
Definition: stream.h:163