1// class template regex -*- C++ -*-
2
3// Copyright (C) 2013-2018 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25/**
26 * @file bits/regex_compiler.tcc
27 * This is an internal header file, included by other library headers.
28 * Do not attempt to use it directly. @headername{regex}
29 */
30
31// FIXME make comments doxygen format.
32
33/*
34// This compiler refers to "Regular Expression Matching Can Be Simple And Fast"
35// (http://swtch.com/~rsc/regexp/regexp1.html),
36// but doesn't strictly follow it.
37//
38// When compiling, states are *chained* instead of tree- or graph-constructed.
39// It's more like structured programs: there's if statement and loop statement.
40//
41// For alternative structure (say "a|b"), aka "if statement", two branches
42// should be constructed. However, these two shall merge to an "end_tag" at
43// the end of this operator:
44//
45// branch1
46// / \
47// => begin_tag end_tag =>
48// \ /
49// branch2
50//
51// This is the difference between this implementation and that in Russ's
52// article.
53//
54// That's why we introduced dummy node here ------ "end_tag" is a dummy node.
55// All dummy nodes will be eliminated at the end of compilation.
56*/
57
58namespace std _GLIBCXX_VISIBILITY(default)
59{
60_GLIBCXX_BEGIN_NAMESPACE_VERSION
61
62namespace __detail
63{
64 template<typename _TraitsT>
65 _Compiler<_TraitsT>::
66 _Compiler(_IterT __b, _IterT __e,
67 const typename _TraitsT::locale_type& __loc, _FlagT __flags)
68 : _M_flags((__flags
69 & (regex_constants::ECMAScript
70 | regex_constants::basic
71 | regex_constants::extended
72 | regex_constants::grep
73 | regex_constants::egrep
74 | regex_constants::awk))
75 ? __flags
76 : __flags | regex_constants::ECMAScript),
77 _M_scanner(__b, __e, _M_flags, __loc),
78 _M_nfa(make_shared<_RegexT>(__loc, _M_flags)),
79 _M_traits(_M_nfa->_M_traits),
80 _M_ctype(std::use_facet<_CtypeT>(__loc))
81 {
82 _StateSeqT __r(*_M_nfa, _M_nfa->_M_start());
83 __r._M_append(_M_nfa->_M_insert_subexpr_begin());
84 this->_M_disjunction();
85 if (!_M_match_token(_ScannerT::_S_token_eof))
86 __throw_regex_error(regex_constants::error_paren);
87 __r._M_append(_M_pop());
88 __glibcxx_assert(_M_stack.empty());
89 __r._M_append(_M_nfa->_M_insert_subexpr_end());
90 __r._M_append(_M_nfa->_M_insert_accept());
91 _M_nfa->_M_eliminate_dummy();
92 }
93
94 template<typename _TraitsT>
95 void
96 _Compiler<_TraitsT>::
97 _M_disjunction()
98 {
99 this->_M_alternative();
100 while (_M_match_token(_ScannerT::_S_token_or))
101 {
102 _StateSeqT __alt1 = _M_pop();
103 this->_M_alternative();
104 _StateSeqT __alt2 = _M_pop();
105 auto __end = _M_nfa->_M_insert_dummy();
106 __alt1._M_append(__end);
107 __alt2._M_append(__end);
108 // __alt2 is state._M_next, __alt1 is state._M_alt. The executor
109 // executes _M_alt before _M_next, as well as executing left
110 // alternative before right one.
111 _M_stack.push(_StateSeqT(*_M_nfa,
112 _M_nfa->_M_insert_alt(
113 __alt2._M_start, __alt1._M_start, false),
114 __end));
115 }
116 }
117
118 template<typename _TraitsT>
119 void
120 _Compiler<_TraitsT>::
121 _M_alternative()
122 {
123 if (this->_M_term())
124 {
125 _StateSeqT __re = _M_pop();
126 this->_M_alternative();
127 __re._M_append(_M_pop());
128 _M_stack.push(__re);
129 }
130 else
131 _M_stack.push(_StateSeqT(*_M_nfa, _M_nfa->_M_insert_dummy()));
132 }
133
134 template<typename _TraitsT>
135 bool
136 _Compiler<_TraitsT>::
137 _M_term()
138 {
139 if (this->_M_assertion())
140 return true;
141 if (this->_M_atom())
142 {
143 while (this->_M_quantifier());
144 return true;
145 }
146 return false;
147 }
148
149 template<typename _TraitsT>
150 bool
151 _Compiler<_TraitsT>::
152 _M_assertion()
153 {
154 if (_M_match_token(_ScannerT::_S_token_line_begin))
155 _M_stack.push(_StateSeqT(*_M_nfa, _M_nfa->_M_insert_line_begin()));
156 else if (_M_match_token(_ScannerT::_S_token_line_end))
157 _M_stack.push(_StateSeqT(*_M_nfa, _M_nfa->_M_insert_line_end()));
158 else if (_M_match_token(_ScannerT::_S_token_word_bound))
159 // _M_value[0] == 'n' means it's negative, say "not word boundary".
160 _M_stack.push(_StateSeqT(*_M_nfa, _M_nfa->
161 _M_insert_word_bound(_M_value[0] == 'n')));
162 else if (_M_match_token(_ScannerT::_S_token_subexpr_lookahead_begin))
163 {
164 auto __neg = _M_value[0] == 'n';
165 this->_M_disjunction();
166 if (!_M_match_token(_ScannerT::_S_token_subexpr_end))
167 __throw_regex_error(regex_constants::error_paren,
168 "Parenthesis is not closed.");
169 auto __tmp = _M_pop();
170 __tmp._M_append(_M_nfa->_M_insert_accept());
171 _M_stack.push(
172 _StateSeqT(
173 *_M_nfa,
174 _M_nfa->_M_insert_lookahead(__tmp._M_start, __neg)));
175 }
176 else
177 return false;
178 return true;
179 }
180
181 template<typename _TraitsT>
182 bool
183 _Compiler<_TraitsT>::
184 _M_quantifier()
185 {
186 bool __neg = (_M_flags & regex_constants::ECMAScript);
187 auto __init = [this, &__neg]()
188 {
189 if (_M_stack.empty())
190 __throw_regex_error(regex_constants::error_badrepeat,
191 "Nothing to repeat before a quantifier.");
192 __neg = __neg && _M_match_token(_ScannerT::_S_token_opt);
193 };
194 if (_M_match_token(_ScannerT::_S_token_closure0))
195 {
196 __init();
197 auto __e = _M_pop();
198 _StateSeqT __r(*_M_nfa,
199 _M_nfa->_M_insert_repeat(_S_invalid_state_id,
200 __e._M_start, __neg));
201 __e._M_append(__r);
202 _M_stack.push(__r);
203 }
204 else if (_M_match_token(_ScannerT::_S_token_closure1))
205 {
206 __init();
207 auto __e = _M_pop();
208 __e._M_append(_M_nfa->_M_insert_repeat(_S_invalid_state_id,
209 __e._M_start, __neg));
210 _M_stack.push(__e);
211 }
212 else if (_M_match_token(_ScannerT::_S_token_opt))
213 {
214 __init();
215 auto __e = _M_pop();
216 auto __end = _M_nfa->_M_insert_dummy();
217 _StateSeqT __r(*_M_nfa,
218 _M_nfa->_M_insert_repeat(_S_invalid_state_id,
219 __e._M_start, __neg));
220 __e._M_append(__end);
221 __r._M_append(__end);
222 _M_stack.push(__r);
223 }
224 else if (_M_match_token(_ScannerT::_S_token_interval_begin))
225 {
226 if (_M_stack.empty())
227 __throw_regex_error(regex_constants::error_badrepeat,
228 "Nothing to repeat before a quantifier.");
229 if (!_M_match_token(_ScannerT::_S_token_dup_count))
230 __throw_regex_error(regex_constants::error_badbrace,
231 "Unexpected token in brace expression.");
232 _StateSeqT __r(_M_pop());
233 _StateSeqT __e(*_M_nfa, _M_nfa->_M_insert_dummy());
234 long __min_rep = _M_cur_int_value(10);
235 bool __infi = false;
236 long __n;
237
238 // {3
239 if (_M_match_token(_ScannerT::_S_token_comma))
240 if (_M_match_token(_ScannerT::_S_token_dup_count)) // {3,7}
241 __n = _M_cur_int_value(10) - __min_rep;
242 else
243 __infi = true;
244 else
245 __n = 0;
246 if (!_M_match_token(_ScannerT::_S_token_interval_end))
247 __throw_regex_error(regex_constants::error_brace,
248 "Unexpected end of brace expression.");
249
250 __neg = __neg && _M_match_token(_ScannerT::_S_token_opt);
251
252 for (long __i = 0; __i < __min_rep; ++__i)
253 __e._M_append(__r._M_clone());
254
255 if (__infi)
256 {
257 auto __tmp = __r._M_clone();
258 _StateSeqT __s(*_M_nfa,
259 _M_nfa->_M_insert_repeat(_S_invalid_state_id,
260 __tmp._M_start, __neg));
261 __tmp._M_append(__s);
262 __e._M_append(__s);
263 }
264 else
265 {
266 if (__n < 0)
267 __throw_regex_error(regex_constants::error_badbrace,
268 "Invalid range in brace expression.");
269 auto __end = _M_nfa->_M_insert_dummy();
270 // _M_alt is the "match more" branch, and _M_next is the
271 // "match less" one. Switch _M_alt and _M_next of all created
272 // nodes. This is a hack but IMO works well.
273 std::stack<_StateIdT> __stack;
274 for (long __i = 0; __i < __n; ++__i)
275 {
276 auto __tmp = __r._M_clone();
277 auto __alt = _M_nfa->_M_insert_repeat(__tmp._M_start,
278 __end, __neg);
279 __stack.push(__alt);
280 __e._M_append(_StateSeqT(*_M_nfa, __alt, __tmp._M_end));
281 }
282 __e._M_append(__end);
283 while (!__stack.empty())
284 {
285 auto& __tmp = (*_M_nfa)[__stack.top()];
286 __stack.pop();
287 std::swap(__tmp._M_next, __tmp._M_alt);
288 }
289 }
290 _M_stack.push(__e);
291 }
292 else
293 return false;
294 return true;
295 }
296
297#define __INSERT_REGEX_MATCHER(__func, ...)\
298 do {\
299 if (!(_M_flags & regex_constants::icase))\
300 if (!(_M_flags & regex_constants::collate))\
301 __func<false, false>(__VA_ARGS__);\
302 else\
303 __func<false, true>(__VA_ARGS__);\
304 else\
305 if (!(_M_flags & regex_constants::collate))\
306 __func<true, false>(__VA_ARGS__);\
307 else\
308 __func<true, true>(__VA_ARGS__);\
309 } while (false)
310
311 template<typename _TraitsT>
312 bool
313 _Compiler<_TraitsT>::
314 _M_atom()
315 {
316 if (_M_match_token(_ScannerT::_S_token_anychar))
317 {
318 if (!(_M_flags & regex_constants::ECMAScript))
319 __INSERT_REGEX_MATCHER(_M_insert_any_matcher_posix);
320 else
321 __INSERT_REGEX_MATCHER(_M_insert_any_matcher_ecma);
322 }
323 else if (_M_try_char())
324 __INSERT_REGEX_MATCHER(_M_insert_char_matcher);
325 else if (_M_match_token(_ScannerT::_S_token_backref))
326 _M_stack.push(_StateSeqT(*_M_nfa, _M_nfa->
327 _M_insert_backref(_M_cur_int_value(10))));
328 else if (_M_match_token(_ScannerT::_S_token_quoted_class))
329 __INSERT_REGEX_MATCHER(_M_insert_character_class_matcher);
330 else if (_M_match_token(_ScannerT::_S_token_subexpr_no_group_begin))
331 {
332 _StateSeqT __r(*_M_nfa, _M_nfa->_M_insert_dummy());
333 this->_M_disjunction();
334 if (!_M_match_token(_ScannerT::_S_token_subexpr_end))
335 __throw_regex_error(regex_constants::error_paren,
336 "Parenthesis is not closed.");
337 __r._M_append(_M_pop());
338 _M_stack.push(__r);
339 }
340 else if (_M_match_token(_ScannerT::_S_token_subexpr_begin))
341 {
342 _StateSeqT __r(*_M_nfa, _M_nfa->_M_insert_subexpr_begin());
343 this->_M_disjunction();
344 if (!_M_match_token(_ScannerT::_S_token_subexpr_end))
345 __throw_regex_error(regex_constants::error_paren,
346 "Parenthesis is not closed.");
347 __r._M_append(_M_pop());
348 __r._M_append(_M_nfa->_M_insert_subexpr_end());
349 _M_stack.push(__r);
350 }
351 else if (!_M_bracket_expression())
352 return false;
353 return true;
354 }
355
356 template<typename _TraitsT>
357 bool
358 _Compiler<_TraitsT>::
359 _M_bracket_expression()
360 {
361 bool __neg =
362 _M_match_token(_ScannerT::_S_token_bracket_neg_begin);
363 if (!(__neg || _M_match_token(_ScannerT::_S_token_bracket_begin)))
364 return false;
365 __INSERT_REGEX_MATCHER(_M_insert_bracket_matcher, __neg);
366 return true;
367 }
368#undef __INSERT_REGEX_MATCHER
369
370 template<typename _TraitsT>
371 template<bool __icase, bool __collate>
372 void
373 _Compiler<_TraitsT>::
374 _M_insert_any_matcher_ecma()
375 {
376 _M_stack.push(_StateSeqT(*_M_nfa,
377 _M_nfa->_M_insert_matcher
378 (_AnyMatcher<_TraitsT, true, __icase, __collate>
379 (_M_traits))));
380 }
381
382 template<typename _TraitsT>
383 template<bool __icase, bool __collate>
384 void
385 _Compiler<_TraitsT>::
386 _M_insert_any_matcher_posix()
387 {
388 _M_stack.push(_StateSeqT(*_M_nfa,
389 _M_nfa->_M_insert_matcher
390 (_AnyMatcher<_TraitsT, false, __icase, __collate>
391 (_M_traits))));
392 }
393
394 template<typename _TraitsT>
395 template<bool __icase, bool __collate>
396 void
397 _Compiler<_TraitsT>::
398 _M_insert_char_matcher()
399 {
400 _M_stack.push(_StateSeqT(*_M_nfa,
401 _M_nfa->_M_insert_matcher
402 (_CharMatcher<_TraitsT, __icase, __collate>
403 (_M_value[0], _M_traits))));
404 }
405
406 template<typename _TraitsT>
407 template<bool __icase, bool __collate>
408 void
409 _Compiler<_TraitsT>::
410 _M_insert_character_class_matcher()
411 {
412 __glibcxx_assert(_M_value.size() == 1);
413 _BracketMatcher<_TraitsT, __icase, __collate> __matcher
414 (_M_ctype.is(_CtypeT::upper, _M_value[0]), _M_traits);
415 __matcher._M_add_character_class(_M_value, false);
416 __matcher._M_ready();
417 _M_stack.push(_StateSeqT(*_M_nfa,
418 _M_nfa->_M_insert_matcher(std::move(__matcher))));
419 }
420
421 template<typename _TraitsT>
422 template<bool __icase, bool __collate>
423 void
424 _Compiler<_TraitsT>::
425 _M_insert_bracket_matcher(bool __neg)
426 {
427 _BracketMatcher<_TraitsT, __icase, __collate> __matcher(__neg, _M_traits);
428 pair<bool, _CharT> __last_char; // Optional<_CharT>
429 __last_char.first = false;
430 if (!(_M_flags & regex_constants::ECMAScript))
431 {
432 if (_M_try_char())
433 {
434 __last_char.first = true;
435 __last_char.second = _M_value[0];
436 }
437 else if (_M_match_token(_ScannerT::_S_token_bracket_dash))
438 {
439 __last_char.first = true;
440 __last_char.second = '-';
441 }
442 }
443 while (_M_expression_term(__last_char, __matcher));
444 if (__last_char.first)
445 __matcher._M_add_char(__last_char.second);
446 __matcher._M_ready();
447 _M_stack.push(_StateSeqT(
448 *_M_nfa,
449 _M_nfa->_M_insert_matcher(std::move(__matcher))));
450 }
451
452 template<typename _TraitsT>
453 template<bool __icase, bool __collate>
454 bool
455 _Compiler<_TraitsT>::
456 _M_expression_term(pair<bool, _CharT>& __last_char,
457 _BracketMatcher<_TraitsT, __icase, __collate>& __matcher)
458 {
459 if (_M_match_token(_ScannerT::_S_token_bracket_end))
460 return false;
461
462 const auto __push_char = [&](_CharT __ch)
463 {
464 if (__last_char.first)
465 __matcher._M_add_char(__last_char.second);
466 else
467 __last_char.first = true;
468 __last_char.second = __ch;
469 };
470 const auto __flush = [&]
471 {
472 if (__last_char.first)
473 {
474 __matcher._M_add_char(__last_char.second);
475 __last_char.first = false;
476 }
477 };
478
479 if (_M_match_token(_ScannerT::_S_token_collsymbol))
480 {
481 auto __symbol = __matcher._M_add_collate_element(_M_value);
482 if (__symbol.size() == 1)
483 __push_char(__symbol[0]);
484 else
485 __flush();
486 }
487 else if (_M_match_token(_ScannerT::_S_token_equiv_class_name))
488 {
489 __flush();
490 __matcher._M_add_equivalence_class(_M_value);
491 }
492 else if (_M_match_token(_ScannerT::_S_token_char_class_name))
493 {
494 __flush();
495 __matcher._M_add_character_class(_M_value, false);
496 }
497 else if (_M_try_char())
498 __push_char(_M_value[0]);
499 // POSIX doesn't allow '-' as a start-range char (say [a-z--0]),
500 // except when the '-' is the first or last character in the bracket
501 // expression ([--0]). ECMAScript treats all '-' after a range as a
502 // normal character. Also see above, where _M_expression_term gets called.
503 //
504 // As a result, POSIX rejects [-----], but ECMAScript doesn't.
505 // Boost (1.57.0) always uses POSIX style even in its ECMAScript syntax.
506 // Clang (3.5) always uses ECMAScript style even in its POSIX syntax.
507 //
508 // It turns out that no one reads BNFs ;)
509 else if (_M_match_token(_ScannerT::_S_token_bracket_dash))
510 {
511 if (!__last_char.first)
512 {
513 if (!(_M_flags & regex_constants::ECMAScript))
514 {
515 if (_M_match_token(_ScannerT::_S_token_bracket_end))
516 {
517 __push_char('-');
518 return false;
519 }
520 __throw_regex_error(
521 regex_constants::error_range,
522 "Unexpected dash in bracket expression. For POSIX syntax, "
523 "a dash is not treated literally only when it is at "
524 "beginning or end.");
525 }
526 __push_char('-');
527 }
528 else
529 {
530 if (_M_try_char())
531 {
532 __matcher._M_make_range(__last_char.second, _M_value[0]);
533 __last_char.first = false;
534 }
535 else if (_M_match_token(_ScannerT::_S_token_bracket_dash))
536 {
537 __matcher._M_make_range(__last_char.second, '-');
538 __last_char.first = false;
539 }
540 else
541 {
542 if (_M_scanner._M_get_token()
543 != _ScannerT::_S_token_bracket_end)
544 __throw_regex_error(
545 regex_constants::error_range,
546 "Character is expected after a dash.");
547 __push_char('-');
548 }
549 }
550 }
551 else if (_M_match_token(_ScannerT::_S_token_quoted_class))
552 {
553 __flush();
554 __matcher._M_add_character_class(_M_value,
555 _M_ctype.is(_CtypeT::upper,
556 _M_value[0]));
557 }
558 else
559 __throw_regex_error(regex_constants::error_brack,
560 "Unexpected character in bracket expression.");
561
562 return true;
563 }
564
565 template<typename _TraitsT>
566 bool
567 _Compiler<_TraitsT>::
568 _M_try_char()
569 {
570 bool __is_char = false;
571 if (_M_match_token(_ScannerT::_S_token_oct_num))
572 {
573 __is_char = true;
574 _M_value.assign(1, _M_cur_int_value(8));
575 }
576 else if (_M_match_token(_ScannerT::_S_token_hex_num))
577 {
578 __is_char = true;
579 _M_value.assign(1, _M_cur_int_value(16));
580 }
581 else if (_M_match_token(_ScannerT::_S_token_ord_char))
582 __is_char = true;
583 return __is_char;
584 }
585
586 template<typename _TraitsT>
587 bool
588 _Compiler<_TraitsT>::
589 _M_match_token(_TokenT token)
590 {
591 if (token == _M_scanner._M_get_token())
592 {
593 _M_value = _M_scanner._M_get_value();
594 _M_scanner._M_advance();
595 return true;
596 }
597 return false;
598 }
599
600 template<typename _TraitsT>
601 int
602 _Compiler<_TraitsT>::
603 _M_cur_int_value(int __radix)
604 {
605 long __v = 0;
606 for (typename _StringT::size_type __i = 0;
607 __i < _M_value.length(); ++__i)
608 __v =__v * __radix + _M_traits.value(_M_value[__i], __radix);
609 return __v;
610 }
611
612 template<typename _TraitsT, bool __icase, bool __collate>
613 bool
614 _BracketMatcher<_TraitsT, __icase, __collate>::
615 _M_apply(_CharT __ch, false_type) const
616 {
617 return [this, __ch]
618 {
619 if (std::binary_search(_M_char_set.begin(), _M_char_set.end(),
620 _M_translator._M_translate(__ch)))
621 return true;
622 auto __s = _M_translator._M_transform(__ch);
623 for (auto& __it : _M_range_set)
624 if (_M_translator._M_match_range(__it.first, __it.second, __s))
625 return true;
626 if (_M_traits.isctype(__ch, _M_class_set))
627 return true;
628 if (std::find(_M_equiv_set.begin(), _M_equiv_set.end(),
629 _M_traits.transform_primary(&__ch, &__ch+1))
630 != _M_equiv_set.end())
631 return true;
632 for (auto& __it : _M_neg_class_set)
633 if (!_M_traits.isctype(__ch, __it))
634 return true;
635 return false;
636 }() ^ _M_is_non_matching;
637 }
638} // namespace __detail
639
640_GLIBCXX_END_NAMESPACE_VERSION
641} // namespace
642