1 | /* |
2 | * Copyright (c) 2015-2019, Intel Corporation |
3 | * |
4 | * Redistribution and use in source and binary forms, with or without |
5 | * modification, are permitted provided that the following conditions are met: |
6 | * |
7 | * * Redistributions of source code must retain the above copyright notice, |
8 | * this list of conditions and the following disclaimer. |
9 | * * Redistributions in binary form must reproduce the above copyright |
10 | * notice, this list of conditions and the following disclaimer in the |
11 | * documentation and/or other materials provided with the distribution. |
12 | * * Neither the name of Intel Corporation nor the names of its contributors |
13 | * may be used to endorse or promote products derived from this software |
14 | * without specific prior written permission. |
15 | * |
16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
17 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
18 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
19 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
20 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
21 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
22 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
23 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
24 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
25 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
26 | * POSSIBILITY OF SUCH DAMAGE. |
27 | */ |
28 | |
29 | #include "catchup.h" |
30 | #include "match.h" |
31 | #include "program_runtime.h" |
32 | #include "rose.h" |
33 | #include "util/bitutils.h" |
34 | #include "util/fatbit.h" |
35 | |
36 | #if defined(DEBUG) || defined(DUMP_SUPPORT) |
37 | #include "util/compare.h" |
38 | /** A debugging crutch: print a hex-escaped version of the match for our |
39 | * perusal. The start and end offsets are stream offsets. */ |
40 | static UNUSED |
41 | void printMatch(const struct core_info *ci, u64a start, u64a end) { |
42 | assert(start <= end); |
43 | assert(end <= ci->buf_offset + ci->len); |
44 | |
45 | printf("'" ); |
46 | u64a i = start; |
47 | for (; i <= MIN(ci->buf_offset, end); i++) { |
48 | u64a h_idx = ci->buf_offset - i; |
49 | u8 c = h_idx >= ci->hlen ? '?' : ci->hbuf[ci->hlen - h_idx - 1]; |
50 | if (ourisprint(c) && c != '\'') { |
51 | printf("%c" , c); |
52 | } else { |
53 | printf("\\x%02x" , c); |
54 | } |
55 | } |
56 | for (; i <= end; i++) { |
57 | u64a b_idx = i - ci->buf_offset - 1; |
58 | u8 c = b_idx >= ci->len ? '?' : ci->buf[b_idx]; |
59 | if (ourisprint(c) && c != '\'') { |
60 | printf("%c" , c); |
61 | } else { |
62 | printf("\\x%02x" , c); |
63 | } |
64 | } |
65 | printf("'" ); |
66 | } |
67 | #endif |
68 | |
69 | hwlmcb_rv_t roseDelayRebuildCallback(size_t end, u32 id, |
70 | struct hs_scratch *scratch) { |
71 | struct RoseContext *tctx = &scratch->tctxt; |
72 | struct core_info *ci = &scratch->core_info; |
73 | const struct RoseEngine *t = ci->rose; |
74 | size_t rb_len = MIN(ci->hlen, t->delayRebuildLength); |
75 | |
76 | u64a real_end = ci->buf_offset - rb_len + end + 1; // index after last byte |
77 | |
78 | #ifdef DEBUG |
79 | DEBUG_PRINTF("REBUILD MATCH id=%u end offset@%llu]: " , id, real_end); |
80 | u64a start = real_end < 8 ? 1 : real_end - 7; |
81 | printMatch(ci, start, real_end); |
82 | printf("\n" ); |
83 | #endif |
84 | |
85 | DEBUG_PRINTF("STATE groups=0x%016llx\n" , tctx->groups); |
86 | |
87 | assert(id && id < t->size); // id is a program offset |
88 | const u64a som = 0; |
89 | const u8 flags = 0; |
90 | UNUSED hwlmcb_rv_t rv = |
91 | roseRunProgram(t, scratch, id, som, real_end, flags); |
92 | assert(rv != HWLM_TERMINATE_MATCHING); |
93 | |
94 | /* we are just repopulating the delay queue, groups should be |
95 | * already set from the original scan. */ |
96 | |
97 | return tctx->groups; |
98 | } |
99 | |
100 | static really_inline |
101 | hwlmcb_rv_t ensureMpvQueueFlushed(const struct RoseEngine *t, |
102 | struct hs_scratch *scratch, u32 qi, s64a loc, |
103 | char in_chained) { |
104 | return ensureQueueFlushed_i(t, scratch, qi, loc, 1, in_chained); |
105 | } |
106 | |
107 | hwlmcb_rv_t roseHandleChainMatch(const struct RoseEngine *t, |
108 | struct hs_scratch *scratch, u32 event, |
109 | u64a top_squash_distance, u64a end, |
110 | char in_catchup) { |
111 | assert(event == MQE_TOP || event >= MQE_TOP_FIRST); |
112 | struct core_info *ci = &scratch->core_info; |
113 | |
114 | u8 *aa = getActiveLeafArray(t, scratch->core_info.state); |
115 | u32 aaCount = t->activeArrayCount; |
116 | struct fatbit *activeQueues = scratch->aqa; |
117 | u32 qCount = t->queueCount; |
118 | |
119 | const u32 qi = 0; /* MPV is always queue 0 if it exists */ |
120 | struct mq *q = &scratch->queues[qi]; |
121 | const struct NfaInfo *info = getNfaInfoByQueue(t, qi); |
122 | |
123 | s64a loc = (s64a)end - ci->buf_offset; |
124 | assert(loc <= (s64a)ci->len && loc >= -(s64a)ci->hlen); |
125 | |
126 | if (!mmbit_set(aa, aaCount, qi)) { |
127 | initQueue(q, qi, t, scratch); |
128 | nfaQueueInitState(q->nfa, q); |
129 | pushQueueAt(q, 0, MQE_START, loc); |
130 | fatbit_set(activeQueues, qCount, qi); |
131 | } else if (info->no_retrigger) { |
132 | DEBUG_PRINTF("yawn\n" ); |
133 | /* nfa only needs one top; we can go home now */ |
134 | return HWLM_CONTINUE_MATCHING; |
135 | } else if (!fatbit_set(activeQueues, qCount, qi)) { |
136 | initQueue(q, qi, t, scratch); |
137 | loadStreamState(q->nfa, q, 0); |
138 | pushQueueAt(q, 0, MQE_START, 0); |
139 | } else if (isQueueFull(q)) { |
140 | DEBUG_PRINTF("queue %u full -> catching up nfas\n" , qi); |
141 | /* we know it is a chained nfa and the suffixes/outfixes must already |
142 | * be known to be consistent */ |
143 | if (ensureMpvQueueFlushed(t, scratch, qi, loc, in_catchup) |
144 | == HWLM_TERMINATE_MATCHING) { |
145 | DEBUG_PRINTF("terminating...\n" ); |
146 | return HWLM_TERMINATE_MATCHING; |
147 | } |
148 | } |
149 | |
150 | if (top_squash_distance) { |
151 | assert(q->cur < q->end); |
152 | struct mq_item *last = &q->items[q->end - 1]; |
153 | if (last->type == event |
154 | && last->location >= loc - (s64a)top_squash_distance) { |
155 | last->location = loc; |
156 | goto event_enqueued; |
157 | } |
158 | } |
159 | |
160 | pushQueue(q, event, loc); |
161 | |
162 | event_enqueued: |
163 | if (q_cur_loc(q) == (s64a)ci->len) { |
164 | /* we may not run the nfa; need to ensure state is fine */ |
165 | DEBUG_PRINTF("empty run\n" ); |
166 | pushQueueNoMerge(q, MQE_END, loc); |
167 | char alive = nfaQueueExec(q->nfa, q, loc); |
168 | if (alive) { |
169 | scratch->tctxt.mpv_inactive = 0; |
170 | q->cur = q->end = 0; |
171 | pushQueueAt(q, 0, MQE_START, loc); |
172 | } else { |
173 | mmbit_unset(aa, aaCount, qi); |
174 | fatbit_unset(scratch->aqa, qCount, qi); |
175 | } |
176 | } |
177 | |
178 | DEBUG_PRINTF("added mpv event at %lld\n" , loc); |
179 | scratch->tctxt.next_mpv_offset = 0; /* the top event may result in matches |
180 | * earlier than expected */ |
181 | return HWLM_CONTINUE_MATCHING; |
182 | } |
183 | |
184 | int roseAnchoredCallback(u64a start, u64a end, u32 id, void *ctx) { |
185 | struct hs_scratch *scratch = ctx; |
186 | assert(scratch && scratch->magic == SCRATCH_MAGIC); |
187 | struct RoseContext *tctxt = &scratch->tctxt; |
188 | struct core_info *ci = &scratch->core_info; |
189 | const struct RoseEngine *t = ci->rose; |
190 | |
191 | u64a real_end = ci->buf_offset + end; // index after last byte |
192 | |
193 | DEBUG_PRINTF("MATCH id=%u offsets=[???,%llu]\n" , id, real_end); |
194 | DEBUG_PRINTF("STATE groups=0x%016llx\n" , tctxt->groups); |
195 | |
196 | if (can_stop_matching(scratch)) { |
197 | DEBUG_PRINTF("received a match when we're already dead!\n" ); |
198 | return MO_HALT_MATCHING; |
199 | } |
200 | |
201 | /* delayed literals need to be delivered before real literals; however |
202 | * delayed literals only come from the floating table so if we are going |
203 | * to deliver a literal here it must be too early for a delayed literal */ |
204 | |
205 | /* no history checks from anchored region and we are before the flush |
206 | * boundary */ |
207 | |
208 | if (real_end <= t->floatingMinLiteralMatchOffset) { |
209 | roseFlushLastByteHistory(t, scratch, real_end); |
210 | tctxt->lastEndOffset = real_end; |
211 | } |
212 | |
213 | // Note that the "id" we have been handed is the program offset. |
214 | const u8 flags = ROSE_PROG_FLAG_IN_ANCHORED; |
215 | if (roseRunProgram(t, scratch, id, start, real_end, flags) |
216 | == HWLM_TERMINATE_MATCHING) { |
217 | assert(can_stop_matching(scratch)); |
218 | DEBUG_PRINTF("caller requested termination\n" ); |
219 | return MO_HALT_MATCHING; |
220 | } |
221 | |
222 | DEBUG_PRINTF("DONE groups=0x%016llx\n" , tctxt->groups); |
223 | |
224 | return MO_CONTINUE_MATCHING; |
225 | } |
226 | |
227 | /** |
228 | * \brief Run the program for the given literal ID, with the interpreter |
229 | * inlined into this call. |
230 | * |
231 | * Assumes not in_anchored. |
232 | */ |
233 | static really_inline |
234 | hwlmcb_rv_t roseProcessMatchInline(const struct RoseEngine *t, |
235 | struct hs_scratch *scratch, u64a end, |
236 | u32 id) { |
237 | DEBUG_PRINTF("id=%u\n" , id); |
238 | assert(id && id < t->size); // id is an offset into bytecode |
239 | const u64a som = 0; |
240 | const u8 flags = 0; |
241 | if (!scratch->pure) { |
242 | return roseRunProgram(t, scratch, id, som, end, flags); |
243 | } else { |
244 | return roseRunProgram_l(t, scratch, id, som, end, flags); |
245 | } |
246 | } |
247 | |
248 | static rose_inline |
249 | hwlmcb_rv_t playDelaySlot(const struct RoseEngine *t, |
250 | struct hs_scratch *scratch, |
251 | struct fatbit **delaySlots, u32 vicIndex, |
252 | u64a offset) { |
253 | /* assert(!tctxt->in_anchored); */ |
254 | assert(vicIndex < DELAY_SLOT_COUNT); |
255 | const struct fatbit *vicSlot = delaySlots[vicIndex]; |
256 | u32 delay_count = t->delay_count; |
257 | |
258 | if (offset < t->floatingMinLiteralMatchOffset) { |
259 | DEBUG_PRINTF("too soon\n" ); |
260 | return HWLM_CONTINUE_MATCHING; |
261 | } |
262 | |
263 | struct RoseContext *tctxt = &scratch->tctxt; |
264 | roseFlushLastByteHistory(t, scratch, offset); |
265 | tctxt->lastEndOffset = offset; |
266 | |
267 | const u32 *programs = getByOffset(t, t->delayProgramOffset); |
268 | |
269 | for (u32 it = fatbit_iterate(vicSlot, delay_count, MMB_INVALID); |
270 | it != MMB_INVALID; it = fatbit_iterate(vicSlot, delay_count, it)) { |
271 | UNUSED rose_group old_groups = tctxt->groups; |
272 | |
273 | DEBUG_PRINTF("DELAYED MATCH id=%u offset=%llu\n" , it, offset); |
274 | const u64a som = 0; |
275 | const u8 flags = 0; |
276 | hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, offset, |
277 | flags); |
278 | DEBUG_PRINTF("DONE groups=0x%016llx\n" , tctxt->groups); |
279 | |
280 | /* delayed literals can't safely set groups. |
281 | * However we may be setting groups that successors already have |
282 | * worked out that we don't need to match the group */ |
283 | DEBUG_PRINTF("groups in %016llx out %016llx\n" , old_groups, |
284 | tctxt->groups); |
285 | |
286 | if (rv == HWLM_TERMINATE_MATCHING) { |
287 | return HWLM_TERMINATE_MATCHING; |
288 | } |
289 | } |
290 | |
291 | return HWLM_CONTINUE_MATCHING; |
292 | } |
293 | |
294 | static really_inline |
295 | hwlmcb_rv_t flushAnchoredLiteralAtLoc(const struct RoseEngine *t, |
296 | struct hs_scratch *scratch, |
297 | u32 curr_loc) { |
298 | struct RoseContext *tctxt = &scratch->tctxt; |
299 | struct fatbit *curr_row = getAnchoredLiteralLog(scratch)[curr_loc - 1]; |
300 | u32 region_width = t->anchored_count; |
301 | |
302 | const u32 *programs = getByOffset(t, t->anchoredProgramOffset); |
303 | |
304 | DEBUG_PRINTF("report matches at curr loc\n" ); |
305 | for (u32 it = fatbit_iterate(curr_row, region_width, MMB_INVALID); |
306 | it != MMB_INVALID; it = fatbit_iterate(curr_row, region_width, it)) { |
307 | DEBUG_PRINTF("it = %u/%u\n" , it, region_width); |
308 | |
309 | rose_group old_groups = tctxt->groups; |
310 | DEBUG_PRINTF("ANCH REPLAY MATCH id=%u offset=%u\n" , it, curr_loc); |
311 | const u64a som = 0; |
312 | const u8 flags = 0; |
313 | hwlmcb_rv_t rv = roseRunProgram(t, scratch, programs[it], som, curr_loc, |
314 | flags); |
315 | DEBUG_PRINTF("DONE groups=0x%016llx\n" , tctxt->groups); |
316 | |
317 | /* anchored literals can't safely set groups. |
318 | * However we may be setting groups that successors already |
319 | * have worked out that we don't need to match the group */ |
320 | DEBUG_PRINTF("groups in %016llx out %016llx\n" , old_groups, |
321 | tctxt->groups); |
322 | tctxt->groups &= old_groups; |
323 | |
324 | if (rv == HWLM_TERMINATE_MATCHING) { |
325 | return HWLM_TERMINATE_MATCHING; |
326 | } |
327 | } |
328 | |
329 | /* clear row; does not invalidate iteration */ |
330 | bf64_unset(&scratch->al_log_sum, curr_loc - 1); |
331 | |
332 | return HWLM_CONTINUE_MATCHING; |
333 | } |
334 | |
335 | static really_inline |
336 | u32 anchored_it_begin(struct hs_scratch *scratch) { |
337 | struct RoseContext *tctxt = &scratch->tctxt; |
338 | if (tctxt->lastEndOffset >= scratch->anchored_literal_region_len) { |
339 | return MMB_INVALID; |
340 | } |
341 | u32 begin = tctxt->lastEndOffset; |
342 | begin--; |
343 | |
344 | return bf64_iterate(scratch->al_log_sum, begin); |
345 | } |
346 | |
347 | static really_inline |
348 | hwlmcb_rv_t flushAnchoredLiterals(const struct RoseEngine *t, |
349 | struct hs_scratch *scratch, |
350 | u32 *anchored_it_param, u64a to_off) { |
351 | struct RoseContext *tctxt = &scratch->tctxt; |
352 | u32 anchored_it = *anchored_it_param; |
353 | /* catch up any remaining anchored matches */ |
354 | for (; anchored_it != MMB_INVALID && anchored_it < to_off; |
355 | anchored_it = bf64_iterate(scratch->al_log_sum, anchored_it)) { |
356 | assert(anchored_it < scratch->anchored_literal_region_len); |
357 | DEBUG_PRINTF("loc_it = %u\n" , anchored_it); |
358 | u32 curr_off = anchored_it + 1; |
359 | roseFlushLastByteHistory(t, scratch, curr_off); |
360 | tctxt->lastEndOffset = curr_off; |
361 | |
362 | if (flushAnchoredLiteralAtLoc(t, scratch, curr_off) |
363 | == HWLM_TERMINATE_MATCHING) { |
364 | return HWLM_TERMINATE_MATCHING; |
365 | } |
366 | } |
367 | |
368 | *anchored_it_param = anchored_it; |
369 | return HWLM_CONTINUE_MATCHING; |
370 | } |
371 | |
372 | static really_inline |
373 | hwlmcb_rv_t playVictims(const struct RoseEngine *t, struct hs_scratch *scratch, |
374 | u32 *anchored_it, u64a lastEnd, u64a victimDelaySlots, |
375 | struct fatbit **delaySlots) { |
376 | while (victimDelaySlots) { |
377 | u32 vic = findAndClearLSB_64(&victimDelaySlots); |
378 | DEBUG_PRINTF("vic = %u\n" , vic); |
379 | u64a vicOffset = vic + (lastEnd & ~(u64a)DELAY_MASK); |
380 | |
381 | if (flushAnchoredLiterals(t, scratch, anchored_it, vicOffset) |
382 | == HWLM_TERMINATE_MATCHING) { |
383 | return HWLM_TERMINATE_MATCHING; |
384 | } |
385 | |
386 | if (playDelaySlot(t, scratch, delaySlots, vic % DELAY_SLOT_COUNT, |
387 | vicOffset) == HWLM_TERMINATE_MATCHING) { |
388 | return HWLM_TERMINATE_MATCHING; |
389 | } |
390 | } |
391 | |
392 | return HWLM_CONTINUE_MATCHING; |
393 | } |
394 | |
395 | /* call flushQueuedLiterals instead */ |
396 | hwlmcb_rv_t flushQueuedLiterals_i(const struct RoseEngine *t, |
397 | struct hs_scratch *scratch, u64a currEnd) { |
398 | struct RoseContext *tctxt = &scratch->tctxt; |
399 | u64a lastEnd = tctxt->delayLastEndOffset; |
400 | DEBUG_PRINTF("flushing backed up matches @%llu up from %llu\n" , currEnd, |
401 | lastEnd); |
402 | |
403 | assert(currEnd != lastEnd); /* checked in main entry point */ |
404 | |
405 | u32 anchored_it = anchored_it_begin(scratch); |
406 | |
407 | if (!tctxt->filledDelayedSlots) { |
408 | DEBUG_PRINTF("no delayed, no flush\n" ); |
409 | goto anchored_leftovers; |
410 | } |
411 | |
412 | { |
413 | struct fatbit **delaySlots = getDelaySlots(scratch); |
414 | |
415 | u32 lastIndex = lastEnd & DELAY_MASK; |
416 | u32 currIndex = currEnd & DELAY_MASK; |
417 | |
418 | int wrapped = (lastEnd | DELAY_MASK) < currEnd; |
419 | |
420 | u64a victimDelaySlots; /* needs to be twice as wide as the number of |
421 | * slots. */ |
422 | |
423 | DEBUG_PRINTF("hello %08x\n" , tctxt->filledDelayedSlots); |
424 | if (!wrapped) { |
425 | victimDelaySlots = tctxt->filledDelayedSlots; |
426 | |
427 | DEBUG_PRINTF("unwrapped %016llx %08x\n" , victimDelaySlots, |
428 | tctxt->filledDelayedSlots); |
429 | /* index vars < 32 so 64bit shifts are safe */ |
430 | |
431 | /* clear all slots at last index and below, */ |
432 | victimDelaySlots &= ~((1LLU << (lastIndex + 1)) - 1); |
433 | |
434 | /* clear all slots above curr index */ |
435 | victimDelaySlots &= (1LLU << (currIndex + 1)) - 1; |
436 | |
437 | tctxt->filledDelayedSlots &= ~victimDelaySlots; |
438 | |
439 | DEBUG_PRINTF("unwrapped %016llx %08x\n" , victimDelaySlots, |
440 | tctxt->filledDelayedSlots); |
441 | } else { |
442 | DEBUG_PRINTF("wrapped %08x\n" , tctxt->filledDelayedSlots); |
443 | |
444 | /* 1st half: clear all slots at last index and below, */ |
445 | u64a first_half = tctxt->filledDelayedSlots; |
446 | first_half &= ~((1ULL << (lastIndex + 1)) - 1); |
447 | tctxt->filledDelayedSlots &= (1ULL << (lastIndex + 1)) - 1; |
448 | |
449 | u64a second_half = tctxt->filledDelayedSlots; |
450 | |
451 | if (currEnd > lastEnd + DELAY_SLOT_COUNT) { |
452 | /* 2nd half: clear all slots above last index */ |
453 | second_half &= (1ULL << (lastIndex + 1)) - 1; |
454 | } else { |
455 | /* 2nd half: clear all slots above curr index */ |
456 | second_half &= (1ULL << (currIndex + 1)) - 1; |
457 | } |
458 | tctxt->filledDelayedSlots &= ~second_half; |
459 | |
460 | victimDelaySlots = first_half | (second_half << DELAY_SLOT_COUNT); |
461 | |
462 | DEBUG_PRINTF("-- %016llx %016llx = %016llx (li %u)\n" , first_half, |
463 | second_half, victimDelaySlots, lastIndex); |
464 | } |
465 | |
466 | if (playVictims(t, scratch, &anchored_it, lastEnd, victimDelaySlots, |
467 | delaySlots) == HWLM_TERMINATE_MATCHING) { |
468 | return HWLM_TERMINATE_MATCHING; |
469 | } |
470 | } |
471 | |
472 | anchored_leftovers:; |
473 | hwlmcb_rv_t rv = flushAnchoredLiterals(t, scratch, &anchored_it, currEnd); |
474 | tctxt->delayLastEndOffset = currEnd; |
475 | return rv; |
476 | } |
477 | |
478 | static really_inline |
479 | hwlmcb_rv_t roseCallback_i(size_t end, u32 id, struct hs_scratch *scratch) { |
480 | struct RoseContext *tctx = &scratch->tctxt; |
481 | const struct RoseEngine *t = scratch->core_info.rose; |
482 | |
483 | u64a real_end = end + tctx->lit_offset_adjust; |
484 | |
485 | #if defined(DEBUG) |
486 | DEBUG_PRINTF("MATCH id=%u end offset@%llu: " , id, real_end); |
487 | u64a start = real_end < 8 ? 1 : real_end - 7; |
488 | printMatch(&scratch->core_info, start, real_end); |
489 | printf("\n" ); |
490 | #endif |
491 | DEBUG_PRINTF("last end %llu\n" , tctx->lastEndOffset); |
492 | |
493 | DEBUG_PRINTF("STATE groups=0x%016llx\n" , tctx->groups); |
494 | |
495 | if (can_stop_matching(scratch)) { |
496 | DEBUG_PRINTF("received a match when we're already dead!\n" ); |
497 | return HWLM_TERMINATE_MATCHING; |
498 | } |
499 | |
500 | hwlmcb_rv_t rv = flushQueuedLiterals(t, scratch, real_end); |
501 | /* flushDelayed may have advanced tctx->lastEndOffset */ |
502 | |
503 | if (real_end >= t->floatingMinLiteralMatchOffset) { |
504 | roseFlushLastByteHistory(t, scratch, real_end); |
505 | tctx->lastEndOffset = real_end; |
506 | } |
507 | |
508 | if (rv == HWLM_TERMINATE_MATCHING) { |
509 | return HWLM_TERMINATE_MATCHING; |
510 | } |
511 | |
512 | rv = roseProcessMatchInline(t, scratch, real_end, id); |
513 | |
514 | DEBUG_PRINTF("DONE groups=0x%016llx\n" , tctx->groups); |
515 | |
516 | if (rv != HWLM_TERMINATE_MATCHING) { |
517 | return tctx->groups; |
518 | } |
519 | |
520 | assert(can_stop_matching(scratch)); |
521 | DEBUG_PRINTF("user requested halt\n" ); |
522 | return HWLM_TERMINATE_MATCHING; |
523 | } |
524 | |
525 | hwlmcb_rv_t roseCallback(size_t end, u32 id, struct hs_scratch *scratch) { |
526 | return roseCallback_i(end, id, scratch); |
527 | } |
528 | |
529 | hwlmcb_rv_t roseFloatingCallback(size_t end, u32 id, |
530 | struct hs_scratch *scratch) { |
531 | const struct RoseEngine *t = scratch->core_info.rose; |
532 | |
533 | return roseCallback_i(end, id, scratch) & t->floating_group_mask; |
534 | } |
535 | |
536 | /** |
537 | * \brief Execute a boundary report program. |
538 | * |
539 | * Returns MO_HALT_MATCHING if the stream is exhausted or the user has |
540 | * instructed us to halt, or MO_CONTINUE_MATCHING otherwise. |
541 | */ |
542 | int roseRunBoundaryProgram(const struct RoseEngine *rose, u32 program, |
543 | u64a stream_offset, struct hs_scratch *scratch) { |
544 | DEBUG_PRINTF("running boundary program at offset %u\n" , program); |
545 | |
546 | if (can_stop_matching(scratch)) { |
547 | DEBUG_PRINTF("can stop matching\n" ); |
548 | return MO_HALT_MATCHING; |
549 | } |
550 | |
551 | if (rose->hasSom && scratch->deduper.current_report_offset == ~0ULL) { |
552 | /* we cannot delay the initialization of the som deduper logs any longer |
553 | * as we are reporting matches. This is done explicitly as we are |
554 | * shortcutting the som handling in the vacuous repeats as we know they |
555 | * all come from non-som patterns. */ |
556 | fatbit_clear(scratch->deduper.som_log[0]); |
557 | fatbit_clear(scratch->deduper.som_log[1]); |
558 | scratch->deduper.som_log_dirty = 0; |
559 | } |
560 | |
561 | // Keep assertions in program report path happy. At offset zero, there can |
562 | // have been no earlier reports. At EOD, all earlier reports should have |
563 | // been handled and we will have been caught up to the stream offset by the |
564 | // time we are running boundary report programs. |
565 | scratch->tctxt.minMatchOffset = stream_offset; |
566 | |
567 | const u64a som = 0; |
568 | const u8 flags = 0; |
569 | hwlmcb_rv_t rv = roseRunProgram(rose, scratch, program, som, stream_offset, |
570 | flags); |
571 | if (rv == HWLM_TERMINATE_MATCHING) { |
572 | return MO_HALT_MATCHING; |
573 | } |
574 | |
575 | return MO_CONTINUE_MATCHING; |
576 | } |
577 | |
578 | /** |
579 | * \brief Execute a flush combination program. |
580 | * |
581 | * Returns MO_HALT_MATCHING if the stream is exhausted or the user has |
582 | * instructed us to halt, or MO_CONTINUE_MATCHING otherwise. |
583 | */ |
584 | int roseRunFlushCombProgram(const struct RoseEngine *rose, |
585 | struct hs_scratch *scratch, u64a end) { |
586 | hwlmcb_rv_t rv = roseRunProgram(rose, scratch, rose->flushCombProgramOffset, |
587 | 0, end, 0); |
588 | if (rv == HWLM_TERMINATE_MATCHING) { |
589 | return MO_HALT_MATCHING; |
590 | } |
591 | return MO_CONTINUE_MATCHING; |
592 | } |
593 | |
594 | int roseReportAdaptor(u64a start, u64a end, ReportID id, void *context) { |
595 | struct hs_scratch *scratch = context; |
596 | assert(scratch && scratch->magic == SCRATCH_MAGIC); |
597 | |
598 | DEBUG_PRINTF("id=%u matched at [%llu,%llu]\n" , id, start, end); |
599 | |
600 | const struct RoseEngine *rose = scratch->core_info.rose; |
601 | |
602 | // Our match ID is the program offset. |
603 | const u32 program = id; |
604 | const u8 flags = ROSE_PROG_FLAG_SKIP_MPV_CATCHUP; |
605 | hwlmcb_rv_t rv = |
606 | roseRunProgram(rose, scratch, program, start, end, flags); |
607 | if (rv == HWLM_TERMINATE_MATCHING) { |
608 | return MO_HALT_MATCHING; |
609 | } |
610 | |
611 | return can_stop_matching(scratch) ? MO_HALT_MATCHING : MO_CONTINUE_MATCHING; |
612 | } |
613 | |