match_at(regex_t* reg, const UChar* str, const UChar* end,
         const UChar* in_right_range, const UChar* sstart, UChar* sprev,
         MatchArg* msa)
{

#if defined(USE_DIRECT_THREADED_CODE)
  static Operation FinishCode[] = { { .opaddr=&&L_FINISH } };
#else
  static Operation FinishCode[] = { { OP_FINISH } };
#endif

#ifdef USE_THREADED_CODE
  static const void *opcode_to_label[] = {
  &&L_FINISH,
  &&L_END,
  &&L_EXACT1,
  &&L_EXACT2,
  &&L_EXACT3,
  &&L_EXACT4,
  &&L_EXACT5,
  &&L_EXACTN,
  &&L_EXACTMB2N1,
  &&L_EXACTMB2N2,
  &&L_EXACTMB2N3,
  &&L_EXACTMB2N,
  &&L_EXACTMB3N,
  &&L_EXACTMBN,
  &&L_EXACT1_IC,
  &&L_EXACTN_IC,
  &&L_CCLASS,
  &&L_CCLASS_MB,
  &&L_CCLASS_MIX,
  &&L_CCLASS_NOT,
  &&L_CCLASS_MB_NOT,
  &&L_CCLASS_MIX_NOT,
  &&L_ANYCHAR,
  &&L_ANYCHAR_ML,
  &&L_ANYCHAR_STAR,
  &&L_ANYCHAR_ML_STAR,
  &&L_ANYCHAR_STAR_PEEK_NEXT,
  &&L_ANYCHAR_ML_STAR_PEEK_NEXT,
  &&L_WORD,
  &&L_WORD_ASCII,
  &&L_NO_WORD,
  &&L_NO_WORD_ASCII,
  &&L_WORD_BOUNDARY,
  &&L_NO_WORD_BOUNDARY,
  &&L_WORD_BEGIN,
  &&L_WORD_END,
  &&L_TEXT_SEGMENT_BOUNDARY,
  &&L_BEGIN_BUF,
  &&L_END_BUF,
  &&L_BEGIN_LINE,
  &&L_END_LINE,
  &&L_SEMI_END_BUF,
  &&L_BEGIN_POSITION,
  &&L_BACKREF1,
  &&L_BACKREF2,
  &&L_BACKREF_N,
  &&L_BACKREF_N_IC,
  &&L_BACKREF_MULTI,
  &&L_BACKREF_MULTI_IC,
  &&L_BACKREF_WITH_LEVEL,
  &&L_BACKREF_WITH_LEVEL_IC,
  &&L_BACKREF_CHECK,
  &&L_BACKREF_CHECK_WITH_LEVEL,
  &&L_MEM_START,
  &&L_MEM_START_PUSH,
  &&L_MEM_END_PUSH,
#ifdef USE_CALL
  &&L_MEM_END_PUSH_REC,
#endif
  &&L_MEM_END,
#ifdef USE_CALL
  &&L_MEM_END_REC,
#endif
  &&L_FAIL,
  &&L_JUMP,
  &&L_PUSH,
  &&L_PUSH_SUPER,
  &&L_POP_OUT,
#ifdef USE_OP_PUSH_OR_JUMP_EXACT
  &&L_PUSH_OR_JUMP_EXACT1,
#endif
  &&L_PUSH_IF_PEEK_NEXT,
  &&L_REPEAT,
  &&L_REPEAT_NG,
  &&L_REPEAT_INC,
  &&L_REPEAT_INC_NG,
  &&L_EMPTY_CHECK_START,
  &&L_EMPTY_CHECK_END,
  &&L_EMPTY_CHECK_END_MEMST,
#ifdef USE_CALL
  &&L_EMPTY_CHECK_END_MEMST_PUSH,
#endif
  &&L_PREC_READ_START,
  &&L_PREC_READ_END,
  &&L_PREC_READ_NOT_START,
  &&L_PREC_READ_NOT_END,
  &&L_ATOMIC_START,
  &&L_ATOMIC_END,
  &&L_LOOK_BEHIND,
  &&L_LOOK_BEHIND_NOT_START,
  &&L_LOOK_BEHIND_NOT_END,
  &&L_PUSH_SAVE_VAL,
  &&L_UPDATE_VAR,
#ifdef USE_CALL
  &&L_CALL,
  &&L_RETURN,
#endif
#ifdef USE_CALLOUT
  &&L_CALLOUT_CONTENTS,
  &&L_CALLOUT_NAME,
#endif
  };
#endif

  int i, n, num_mem, best_len, pop_level;
  LengthType tlen, tlen2;
  MemNumType mem;
  RelAddrType addr;
  UChar *s, *q, *ps, *sbegin;
  UChar *right_range;
  int is_alloca;
  char *alloc_base;
  StackType *stk_base, *stk, *stk_end;
  StackType *stkp; /* used as any purpose. */
  StackIndex *mem_start_stk, *mem_end_stk;
  UChar* keep;

#ifdef USE_REPEAT_AND_EMPTY_CHECK_LOCAL_VAR
  StackIndex *repeat_stk;
  StackIndex *empty_check_stk;
#endif
#ifdef USE_RETRY_LIMIT_IN_MATCH
  unsigned long retry_limit_in_match;
  unsigned long retry_in_match_counter;
#endif
#ifdef USE_CALLOUT
  int of;
#endif

  Operation* p = reg->ops;
  OnigOptionType option = reg->options;
  OnigEncoding encode = reg->enc;
  OnigCaseFoldType case_fold_flag = reg->case_fold_flag;

#ifdef ONIG_DEBUG_MATCH
  static unsigned int counter = 1;
#endif

#ifdef USE_DIRECT_THREADED_CODE
  if (IS_NULL(msa)) {
    for (i = 0; i < reg->ops_used; i++) {
       const void* addr;
       addr = opcode_to_label[reg->ocs[i]];
       p->opaddr = addr;
       p++;
    }
    return ONIG_NORMAL;
  }
#endif

#ifdef USE_CALLOUT
  msa->mp->match_at_call_counter++;
#endif

#ifdef USE_RETRY_LIMIT_IN_MATCH
  retry_limit_in_match = msa->retry_limit_in_match;
#endif

  pop_level = reg->stack_pop_level;
  num_mem = reg->num_mem;
  STACK_INIT(INIT_MATCH_STACK_SIZE);
  UPDATE_FOR_STACK_REALLOC;
  for (i = 1; i <= num_mem; i++) {
    mem_start_stk[i] = mem_end_stk[i] = INVALID_STACK_INDEX;
  }

#ifdef ONIG_DEBUG_MATCH
  fprintf(stderr, "match_at: str: %p, end: %p, start: %p, sprev: %p\n",
          str, end, sstart, sprev);
  fprintf(stderr, "size: %d, start offset: %d\n",
          (int )(end - str), (int )(sstart - str));
#endif

  best_len = ONIG_MISMATCH;
  keep = s = (UChar* )sstart;
  STACK_PUSH_BOTTOM(STK_ALT, FinishCode);  /* bottom stack */
  INIT_RIGHT_RANGE;

#ifdef USE_RETRY_LIMIT_IN_MATCH
  retry_in_match_counter = 0;
#endif

  BYTECODE_INTERPRETER_START {
    CASE_OP(END)
      n = (int )(s - sstart);
      if (n > best_len) {
        OnigRegion* region;
#ifdef USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE
        if (IS_FIND_LONGEST(option)) {
          if (n > msa->best_len) {
            msa->best_len = n;
            msa->best_s   = (UChar* )sstart;
          }
          else
            goto end_best_len;
        }
#endif
        best_len = n;
        region = msa->region;
        if (region) {
          if (keep > s) keep = s;

#ifdef USE_POSIX_API_REGION_OPTION
          if (IS_POSIX_REGION(msa->options)) {
            posix_regmatch_t* rmt = (posix_regmatch_t* )region;

            rmt[0].rm_so = (regoff_t )(keep - str);
            rmt[0].rm_eo = (regoff_t )(s    - str);
            for (i = 1; i <= num_mem; i++) {
              if (mem_end_stk[i] != INVALID_STACK_INDEX) {
                rmt[i].rm_so = (regoff_t )(STACK_MEM_START(reg, i) - str);
                rmt[i].rm_eo = (regoff_t )(STACK_MEM_END(reg, i)   - str);
              }
              else {
                rmt[i].rm_so = rmt[i].rm_eo = ONIG_REGION_NOTPOS;
              }
            }
          }
          else {
#endif /* USE_POSIX_API_REGION_OPTION */
            region->beg[0] = (int )(keep - str);
            region->end[0] = (int )(s    - str);
            for (i = 1; i <= num_mem; i++) {
              if (mem_end_stk[i] != INVALID_STACK_INDEX) {
                region->beg[i] = (int )(STACK_MEM_START(reg, i) - str);
                region->end[i] = (int )(STACK_MEM_END(reg, i)   - str);
              }
              else {
                region->beg[i] = region->end[i] = ONIG_REGION_NOTPOS;
              }
            }

#ifdef USE_CAPTURE_HISTORY
            if (reg->capture_history != 0) {
              int r;
              OnigCaptureTreeNode* node;

              if (IS_NULL(region->history_root)) {
                region->history_root = node = history_node_new();
                CHECK_NULL_RETURN_MEMERR(node);
              }
              else {
                node = region->history_root;
                history_tree_clear(node);
              }

              node->group = 0;
              node->beg   = (int )(keep - str);
              node->end   = (int )(s    - str);

              stkp = stk_base;
              r = make_capture_history_tree(region->history_root, &stkp,
                                            stk, (UChar* )str, reg);
              if (r < 0) MATCH_AT_ERROR_RETURN(r);
            }
#endif /* USE_CAPTURE_HISTORY */
#ifdef USE_POSIX_API_REGION_OPTION
          } /* else IS_POSIX_REGION() */
#endif
        } /* if (region) */
      } /* n > best_len */

#ifdef USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE
    end_best_len:
#endif
      SOP_OUT;

      if (IS_FIND_CONDITION(option)) {
        if (IS_FIND_NOT_EMPTY(option) && s == sstart) {
          best_len = ONIG_MISMATCH;
          goto fail; /* for retry */
        }
        if (IS_FIND_LONGEST(option) && DATA_ENSURE_CHECK1) {
          goto fail; /* for retry */
        }
      }

      /* default behavior: return first-matching result. */
      goto match_at_end;

    CASE_OP(EXACT1)
      DATA_ENSURE(1);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      s++;
      INC_OP;
      NEXT_OUT;

    CASE_OP(EXACT1_IC)
      {
        int len;
        UChar *q, lowbuf[ONIGENC_MBC_CASE_FOLD_MAXLEN];

        DATA_ENSURE(1);
        len = ONIGENC_MBC_CASE_FOLD(encode,
                 /* DISABLE_CASE_FOLD_MULTI_CHAR(case_fold_flag), */
                                    case_fold_flag,
                                    &s, end, lowbuf);
        DATA_ENSURE(0);
        q = lowbuf;
        ps = p->exact.s;
        while (len-- > 0) {
          if (*ps != *q) goto fail;
          ps++; q++;
        }
      }
      INC_OP;
      NEXT_OUT;

    CASE_OP(EXACT2)
      DATA_ENSURE(2);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      sprev = s;
      s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACT3)
      DATA_ENSURE(3);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      sprev = s;
      s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACT4)
      DATA_ENSURE(4);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      sprev = s;
      s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACT5)
      DATA_ENSURE(5);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      sprev = s;
      s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTN)
      tlen = p->exact_n.n;
      DATA_ENSURE(tlen);
      ps = p->exact_n.s;
      while (tlen-- > 0) {
        if (*ps++ != *s++) goto fail;
      }
      sprev = s - 1;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTN_IC)
      {
        int len;
        UChar *q, *endp, lowbuf[ONIGENC_MBC_CASE_FOLD_MAXLEN];

        tlen = p->exact_n.n;
        ps   = p->exact_n.s;
        endp = ps + tlen;
        while (ps < endp) {
          sprev = s;
          DATA_ENSURE(1);
          len = ONIGENC_MBC_CASE_FOLD(encode,
                        /* DISABLE_CASE_FOLD_MULTI_CHAR(case_fold_flag), */
                                      case_fold_flag,
                                      &s, end, lowbuf);
          DATA_ENSURE(0);
          q = lowbuf;
          while (len-- > 0) {
            if (ps >= endp) goto fail;
            if (*ps != *q) goto fail;
            ps++; q++;
          }
        }
      }

      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTMB2N1)
      DATA_ENSURE(2);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      s++;
      INC_OP;
      NEXT_OUT;

    CASE_OP(EXACTMB2N2)
      DATA_ENSURE(4);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      sprev = s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTMB2N3)
      DATA_ENSURE(6);
      ps = p->exact.s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      sprev = s;
      if (*ps != *s) goto fail;
      ps++; s++;
      if (*ps != *s) goto fail;
      ps++; s++;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTMB2N)
      tlen = p->exact_n.n;
      DATA_ENSURE(tlen * 2);
      ps = p->exact_n.s;
      while (tlen-- > 0) {
        if (*ps != *s) goto fail;
        ps++; s++;
        if (*ps != *s) goto fail;
        ps++; s++;
      }
      sprev = s - 2;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTMB3N)
      tlen = p->exact_n.n;
      DATA_ENSURE(tlen * 3);
      ps = p->exact_n.s;
      while (tlen-- > 0) {
        if (*ps != *s) goto fail;
        ps++; s++;
        if (*ps != *s) goto fail;
        ps++; s++;
        if (*ps != *s) goto fail;
        ps++; s++;
      }
      sprev = s - 3;
      INC_OP;
      JUMP_OUT;

    CASE_OP(EXACTMBN)
      tlen  = p->exact_len_n.len; /* mb byte len */
      tlen2 = p->exact_len_n.n;   /* number of chars */
      tlen2 *= tlen;
      DATA_ENSURE(tlen2);
      ps = p->exact_len_n.s;
      while (tlen2-- > 0) {
        if (*ps != *s) goto fail;
        ps++; s++;
      }
      sprev = s - tlen;
      INC_OP;
      JUMP_OUT;

    CASE_OP(CCLASS)
      DATA_ENSURE(1);
      if (BITSET_AT(p->cclass.bsp, *s) == 0) goto fail;
      s++;
      INC_OP;
      NEXT_OUT;

    CASE_OP(CCLASS_MB)
      DATA_ENSURE(1);
      if (! ONIGENC_IS_MBC_HEAD(encode, s)) goto fail;

    cclass_mb:
      {
        OnigCodePoint code;
        UChar *ss;
        int mb_len;

        mb_len = enclen(encode, s);
        DATA_ENSURE(mb_len);
        ss = s;
        s += mb_len;
        code = ONIGENC_MBC_TO_CODE(encode, ss, s);
        if (! onig_is_in_code_range(p->cclass_mb.mb, code)) goto fail;
      }
      INC_OP;
      NEXT_OUT;

    CASE_OP(CCLASS_MIX)
      DATA_ENSURE(1);
      if (ONIGENC_IS_MBC_HEAD(encode, s)) {
        goto cclass_mb;
      }
      else {
        if (BITSET_AT(p->cclass_mix.bsp, *s) == 0)
          goto fail;

        s++;
      }
      INC_OP;
      NEXT_OUT;

    CASE_OP(CCLASS_NOT)
      DATA_ENSURE(1);
      if (BITSET_AT(p->cclass.bsp, *s) != 0) goto fail;
      s += enclen(encode, s);
      INC_OP;
      NEXT_OUT;

    CASE_OP(CCLASS_MB_NOT)
      DATA_ENSURE(1);
      if (! ONIGENC_IS_MBC_HEAD(encode, s)) {
        s++;
        goto cc_mb_not_success;
      }

    cclass_mb_not:
      {
        OnigCodePoint code;
        UChar *ss;
        int mb_len = enclen(encode, s);

        if (! DATA_ENSURE_CHECK(mb_len)) {
          DATA_ENSURE(1);
          s = (UChar* )end;
          goto cc_mb_not_success;
        }

        ss = s;
        s += mb_len;
        code = ONIGENC_MBC_TO_CODE(encode, ss, s);
        if (onig_is_in_code_range(p->cclass_mb.mb, code)) goto fail;
      }

    cc_mb_not_success:
      INC_OP;
      NEXT_OUT;

    CASE_OP(CCLASS_MIX_NOT)
      DATA_ENSURE(1);
      if (ONIGENC_IS_MBC_HEAD(encode, s)) {
        goto cclass_mb_not;
      }
      else {
        if (BITSET_AT(p->cclass_mix.bsp, *s) != 0)
          goto fail;

        s++;
      }
      INC_OP;
      NEXT_OUT;

    CASE_OP(ANYCHAR)
      DATA_ENSURE(1);
      n = enclen(encode, s);
      DATA_ENSURE(n);
      if (ONIGENC_IS_MBC_NEWLINE(encode, s, end)) goto fail;
      s += n;
      INC_OP;
      NEXT_OUT;

    CASE_OP(ANYCHAR_ML)
      DATA_ENSURE(1);
      n = enclen(encode, s);
      DATA_ENSURE(n);
      s += n;
      INC_OP;
      NEXT_OUT;

    CASE_OP(ANYCHAR_STAR)
      INC_OP;
      while (DATA_ENSURE_CHECK1) {
        STACK_PUSH_ALT(p, s, sprev);
        n = enclen(encode, s);
        DATA_ENSURE(n);
        if (ONIGENC_IS_MBC_NEWLINE(encode, s, end))  goto fail;
        sprev = s;
        s += n;
      }
      JUMP_OUT;

    CASE_OP(ANYCHAR_ML_STAR)
      INC_OP;
      while (DATA_ENSURE_CHECK1) {
        STACK_PUSH_ALT(p, s, sprev);
        n = enclen(encode, s);
        if (n > 1) {
          DATA_ENSURE(n);
          sprev = s;
          s += n;
        }
        else {
          sprev = s;
          s++;
        }
      }
      JUMP_OUT;

    CASE_OP(ANYCHAR_STAR_PEEK_NEXT)
      {
        UChar c;

        c = p->anychar_star_peek_next.c;
        INC_OP;
        while (DATA_ENSURE_CHECK1) {
          if (c == *s) {
            STACK_PUSH_ALT(p, s, sprev);
          }
          n = enclen(encode, s);
          DATA_ENSURE(n);
          if (ONIGENC_IS_MBC_NEWLINE(encode, s, end))  goto fail;
          sprev = s;
          s += n;
        }
      }
      NEXT_OUT;

    CASE_OP(ANYCHAR_ML_STAR_PEEK_NEXT)
      {
        UChar c;

        c = p->anychar_star_peek_next.c;
        INC_OP;
        while (DATA_ENSURE_CHECK1) {
          if (c == *s) {
            STACK_PUSH_ALT(p, s, sprev);
          }
          n = enclen(encode, s);
          if (n > 1) {
            DATA_ENSURE(n);
            sprev = s;
            s += n;
          }
          else {
            sprev = s;
            s++;
          }
        }
      }
      NEXT_OUT;

    CASE_OP(WORD)
      DATA_ENSURE(1);
      if (! ONIGENC_IS_MBC_WORD(encode, s, end))
        goto fail;

      s += enclen(encode, s);
      INC_OP;
      NEXT_OUT;

    CASE_OP(WORD_ASCII)
      DATA_ENSURE(1);
      if (! ONIGENC_IS_MBC_WORD_ASCII(encode, s, end))
        goto fail;

      s += enclen(encode, s);
      INC_OP;
      NEXT_OUT;

    CASE_OP(NO_WORD)
      DATA_ENSURE(1);
      if (ONIGENC_IS_MBC_WORD(encode, s, end))
        goto fail;

      s += enclen(encode, s);
      INC_OP;
      NEXT_OUT;

    CASE_OP(NO_WORD_ASCII)
      DATA_ENSURE(1);
      if (ONIGENC_IS_MBC_WORD_ASCII(encode, s, end))
        goto fail;

      s += enclen(encode, s);
      INC_OP;
      NEXT_OUT;

    CASE_OP(WORD_BOUNDARY)
      {
        ModeType mode;

        mode = p->word_boundary.mode;
        if (ON_STR_BEGIN(s)) {
          DATA_ENSURE(1);
          if (! IS_MBC_WORD_ASCII_MODE(encode, s, end, mode))
            goto fail;
        }
        else if (ON_STR_END(s)) {
          if (! IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode))
            goto fail;
        }
        else {
          if (IS_MBC_WORD_ASCII_MODE(encode, s, end, mode)
              == IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode))
            goto fail;
        }
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(NO_WORD_BOUNDARY)
      {
        ModeType mode;

        mode = p->word_boundary.mode;
        if (ON_STR_BEGIN(s)) {
          if (DATA_ENSURE_CHECK1 && IS_MBC_WORD_ASCII_MODE(encode, s, end, mode))
            goto fail;
        }
        else if (ON_STR_END(s)) {
          if (IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode))
            goto fail;
        }
        else {
          if (IS_MBC_WORD_ASCII_MODE(encode, s, end, mode)
              != IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode))
            goto fail;
        }
      }
      INC_OP;
      JUMP_OUT;

#ifdef USE_WORD_BEGIN_END
    CASE_OP(WORD_BEGIN)
      {
        ModeType mode;

        mode = p->word_boundary.mode;
        if (DATA_ENSURE_CHECK1 && IS_MBC_WORD_ASCII_MODE(encode, s, end, mode)) {
          if (ON_STR_BEGIN(s) || !IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode)) {
            INC_OP;
            JUMP_OUT;
          }
        }
      }
      goto fail;

    CASE_OP(WORD_END)
      {
        ModeType mode;

        mode = p->word_boundary.mode;
        if (!ON_STR_BEGIN(s) && IS_MBC_WORD_ASCII_MODE(encode, sprev, end, mode)) {
          if (ON_STR_END(s) || ! IS_MBC_WORD_ASCII_MODE(encode, s, end, mode)) {
            INC_OP;
            JUMP_OUT;
          }
        }
      }
      goto fail;
#endif

    CASE_OP(TEXT_SEGMENT_BOUNDARY)
      {
        int is_break;

        switch (p->text_segment_boundary.type) {
        case EXTENDED_GRAPHEME_CLUSTER_BOUNDARY:
          is_break = onigenc_egcb_is_break_position(encode, s, sprev, str, end);
          break;
#ifdef USE_UNICODE_WORD_BREAK
        case WORD_BOUNDARY:
          is_break = onigenc_wb_is_break_position(encode, s, sprev, str, end);
          break;
#endif
        default:
          MATCH_AT_ERROR_RETURN(ONIGERR_UNDEFINED_BYTECODE);
          break;
        }

        if (p->text_segment_boundary.not != 0)
          is_break = ! is_break;

        if (is_break != 0) {
          INC_OP;
          JUMP_OUT;
        }
        else {
          goto fail;
        }
      }

    CASE_OP(BEGIN_BUF)
      if (! ON_STR_BEGIN(s)) goto fail;

      INC_OP;
      JUMP_OUT;

    CASE_OP(END_BUF)
      if (! ON_STR_END(s)) goto fail;

      INC_OP;
      JUMP_OUT;

    CASE_OP(BEGIN_LINE)
      if (ON_STR_BEGIN(s)) {
        if (IS_NOTBOL(msa->options)) goto fail;
        INC_OP;
        JUMP_OUT;
      }
      else if (ONIGENC_IS_MBC_NEWLINE(encode, sprev, end) && !ON_STR_END(s)) {
        INC_OP;
        JUMP_OUT;
      }
      goto fail;

    CASE_OP(END_LINE)
      if (ON_STR_END(s)) {
#ifndef USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE
        if (IS_EMPTY_STR || !ONIGENC_IS_MBC_NEWLINE(encode, sprev, end)) {
#endif
          if (IS_NOTEOL(msa->options)) goto fail;
          INC_OP;
          JUMP_OUT;
#ifndef USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE
        }
#endif
      }
      else if (ONIGENC_IS_MBC_NEWLINE(encode, s, end)) {
        INC_OP;
        JUMP_OUT;
      }
#ifdef USE_CRNL_AS_LINE_TERMINATOR
      else if (ONIGENC_IS_MBC_CRNL(encode, s, end)) {
        INC_OP;
        JUMP_OUT;
      }
#endif
      goto fail;

    CASE_OP(SEMI_END_BUF)
      if (ON_STR_END(s)) {
#ifndef USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE
        if (IS_EMPTY_STR || !ONIGENC_IS_MBC_NEWLINE(encode, sprev, end)) {
#endif
          if (IS_NOTEOL(msa->options)) goto fail;
          INC_OP;
          JUMP_OUT;
#ifndef USE_NEWLINE_AT_END_OF_STRING_HAS_EMPTY_LINE
        }
#endif
      }
      else if (ONIGENC_IS_MBC_NEWLINE(encode, s, end) &&
               ON_STR_END(s + enclen(encode, s))) {
        INC_OP;
        JUMP_OUT;
      }
#ifdef USE_CRNL_AS_LINE_TERMINATOR
      else if (ONIGENC_IS_MBC_CRNL(encode, s, end)) {
        UChar* ss = s + enclen(encode, s);
        ss += enclen(encode, ss);
        if (ON_STR_END(ss)) {
          INC_OP;
          JUMP_OUT;
        }
      }
#endif
      goto fail;

    CASE_OP(BEGIN_POSITION)
      if (s != msa->start)
        goto fail;

      INC_OP;
      JUMP_OUT;

    CASE_OP(MEM_START_PUSH)
      mem = p->memory_start.num;
      STACK_PUSH_MEM_START(mem, s);
      INC_OP;
      JUMP_OUT;

    CASE_OP(MEM_START)
      mem = p->memory_start.num;
      mem_start_stk[mem] = (StackIndex )((void* )s);
      INC_OP;
      JUMP_OUT;

    CASE_OP(MEM_END_PUSH)
      mem = p->memory_end.num;
      STACK_PUSH_MEM_END(mem, s);
      INC_OP;
      JUMP_OUT;

    CASE_OP(MEM_END)
      mem = p->memory_end.num;
      mem_end_stk[mem] = (StackIndex )((void* )s);
      INC_OP;
      JUMP_OUT;

#ifdef USE_CALL
    CASE_OP(MEM_END_PUSH_REC)
      {
        StackIndex si;

        mem = p->memory_end.num;
        STACK_GET_MEM_START(mem, stkp); /* should be before push mem-end. */
        si = GET_STACK_INDEX(stkp);
        STACK_PUSH_MEM_END(mem, s);
        mem_start_stk[mem] = si;
        INC_OP;
        JUMP_OUT;
      }

    CASE_OP(MEM_END_REC)
      mem = p->memory_end.num;
      mem_end_stk[mem] = (StackIndex )((void* )s);
      STACK_GET_MEM_START(mem, stkp);

      if (MEM_STATUS_AT(reg->push_mem_start, mem))
        mem_start_stk[mem] = GET_STACK_INDEX(stkp);
      else
        mem_start_stk[mem] = (StackIndex )((void* )stkp->u.mem.pstr);

      STACK_PUSH_MEM_END_MARK(mem);
      INC_OP;
      JUMP_OUT;
#endif

    CASE_OP(BACKREF1)
      mem = 1;
      goto backref;

    CASE_OP(BACKREF2)
      mem = 2;
      goto backref;

    CASE_OP(BACKREF_N)
      mem = p->backref_n.n1;
    backref:
      {
        int len;
        UChar *pstart, *pend;

        if (mem_end_stk[mem]   == INVALID_STACK_INDEX) goto fail;
        if (mem_start_stk[mem] == INVALID_STACK_INDEX) goto fail;

        pstart = STACK_MEM_START(reg, mem);
        pend   = STACK_MEM_END(reg, mem);
        n = (int )(pend - pstart);
        if (n != 0) {
          DATA_ENSURE(n);
          sprev = s;
          STRING_CMP(s, pstart, n);
          while (sprev + (len = enclen(encode, sprev)) < s)
            sprev += len;
        }
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(BACKREF_N_IC)
      mem = p->backref_n.n1;
      {
        int len;
        UChar *pstart, *pend;

        if (mem_end_stk[mem]   == INVALID_STACK_INDEX) goto fail;
        if (mem_start_stk[mem] == INVALID_STACK_INDEX) goto fail;

        pstart = STACK_MEM_START(reg, mem);
        pend   = STACK_MEM_END(reg, mem);
        n = (int )(pend - pstart);
        if (n != 0) {
          DATA_ENSURE(n);
          sprev = s;
          STRING_CMP_IC(case_fold_flag, pstart, &s, n);
          while (sprev + (len = enclen(encode, sprev)) < s)
            sprev += len;
        }
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(BACKREF_MULTI)
      {
        int len, is_fail;
        UChar *pstart, *pend, *swork;

        tlen = p->backref_general.num;
        for (i = 0; i < tlen; i++) {
          mem = tlen == 1 ? p->backref_general.n1 : p->backref_general.ns[i];

          if (mem_end_stk[mem]   == INVALID_STACK_INDEX) continue;
          if (mem_start_stk[mem] == INVALID_STACK_INDEX) continue;

          pstart = STACK_MEM_START(reg, mem);
          pend   = STACK_MEM_END(reg, mem);
          n = (int )(pend - pstart);
          if (n != 0) {
            DATA_ENSURE(n);
            sprev = s;
            swork = s;
            STRING_CMP_VALUE(swork, pstart, n, is_fail);
            if (is_fail) continue;
            s = swork;
            while (sprev + (len = enclen(encode, sprev)) < s)
              sprev += len;
          }
          break; /* success */
        }
        if (i == tlen) goto fail;
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(BACKREF_MULTI_IC)
      {
        int len, is_fail;
        UChar *pstart, *pend, *swork;

        tlen = p->backref_general.num;
        for (i = 0; i < tlen; i++) {
          mem = tlen == 1 ? p->backref_general.n1 : p->backref_general.ns[i];

          if (mem_end_stk[mem]   == INVALID_STACK_INDEX) continue;
          if (mem_start_stk[mem] == INVALID_STACK_INDEX) continue;

          pstart = STACK_MEM_START(reg, mem);
          pend   = STACK_MEM_END(reg, mem);
          n = (int )(pend - pstart);
          if (n != 0) {
            DATA_ENSURE(n);
            sprev = s;
            swork = s;
            STRING_CMP_VALUE_IC(case_fold_flag, pstart, &swork, n, is_fail);
            if (is_fail) continue;
            s = swork;
            while (sprev + (len = enclen(encode, sprev)) < s)
              sprev += len;
          }
          break; /* success */
        }
        if (i == tlen) goto fail;
      }
      INC_OP;
      JUMP_OUT;

#ifdef USE_BACKREF_WITH_LEVEL
    CASE_OP(BACKREF_WITH_LEVEL_IC)
      n = 1; /* ignore case */
      goto backref_with_level;
    CASE_OP(BACKREF_WITH_LEVEL)
      {
        int len;
        int level;
        MemNumType* mems;
        UChar* ssave;

        n = 0;
      backref_with_level:
        level = p->backref_general.nest_level;
        tlen  = p->backref_general.num;
        mems = tlen == 1 ? &(p->backref_general.n1) : p->backref_general.ns;

        ssave = s;
        if (backref_match_at_nested_level(reg, stk, stk_base, n,
                    case_fold_flag, level, (int )tlen, mems, &s, end)) {
          if (ssave != s) {
            sprev = ssave;
            while (sprev + (len = enclen(encode, sprev)) < s)
              sprev += len;
          }
        }
        else
          goto fail;
      }
      INC_OP;
      JUMP_OUT;
#endif

    CASE_OP(BACKREF_CHECK)
      {
        MemNumType* mems;

        tlen  = p->backref_general.num;
        mems = tlen == 1 ? &(p->backref_general.n1) : p->backref_general.ns;

        for (i = 0; i < tlen; i++) {
          mem = mems[i];
          if (mem_end_stk[mem]   == INVALID_STACK_INDEX) continue;
          if (mem_start_stk[mem] == INVALID_STACK_INDEX) continue;
          break; /* success */
        }
        if (i == tlen) goto fail;
      }
      INC_OP;
      JUMP_OUT;

#ifdef USE_BACKREF_WITH_LEVEL
    CASE_OP(BACKREF_CHECK_WITH_LEVEL)
      {
        LengthType level;
        MemNumType* mems;

        level = p->backref_general.nest_level;
        tlen  = p->backref_general.num;
        mems = tlen == 1 ? &(p->backref_general.n1) : p->backref_general.ns;

        if (backref_check_at_nested_level(reg, stk, stk_base,
                                          (int )level, (int )tlen, mems) == 0)
          goto fail;
      }
      INC_OP;
      JUMP_OUT;
#endif

    CASE_OP(EMPTY_CHECK_START)
      mem = p->empty_check_start.mem;   /* mem: null check id */
      STACK_PUSH_EMPTY_CHECK_START(mem, s);
      INC_OP;
      JUMP_OUT;

    CASE_OP(EMPTY_CHECK_END)
      {
        int is_empty;

        mem = p->empty_check_end.mem;  /* mem: null check id */
        STACK_EMPTY_CHECK(is_empty, mem, s);
        INC_OP;
        if (is_empty) {
#ifdef ONIG_DEBUG_MATCH
          fprintf(stderr, "EMPTY_CHECK_END: skip  id:%d, s:%p\n", (int )mem, s);
#endif
        empty_check_found:
          /* empty loop founded, skip next instruction */
#if defined(ONIG_DEBUG) && !defined(USE_DIRECT_THREADED_CODE)
          switch (p->opcode) {
          case OP_JUMP:
          case OP_PUSH:
          case OP_REPEAT_INC:
          case OP_REPEAT_INC_NG:
            INC_OP;
            break;
          default:
            MATCH_AT_ERROR_RETURN(ONIGERR_UNEXPECTED_BYTECODE);
            break;
          }
#else
          INC_OP;
#endif
        }
      }
      JUMP_OUT;

#ifdef USE_STUBBORN_CHECK_CAPTURES_IN_EMPTY_REPEAT
    CASE_OP(EMPTY_CHECK_END_MEMST)
      {
        int is_empty;

        mem = p->empty_check_end.mem;  /* mem: null check id */
        STACK_EMPTY_CHECK_MEM(is_empty, mem, s, reg);
        INC_OP;
        if (is_empty) {
#ifdef ONIG_DEBUG_MATCH
          fprintf(stderr, "EMPTY_CHECK_END_MEM: skip  id:%d, s:%p\n", (int)mem, s);
#endif
          if (is_empty == -1) goto fail;
          goto empty_check_found;
        }
      }
      JUMP_OUT;
#endif

#ifdef USE_CALL
    CASE_OP(EMPTY_CHECK_END_MEMST_PUSH)
      {
        int is_empty;

        mem = p->empty_check_end.mem;  /* mem: null check id */
#ifdef USE_STUBBORN_CHECK_CAPTURES_IN_EMPTY_REPEAT
        STACK_EMPTY_CHECK_MEM_REC(is_empty, mem, s, reg);
#else
        STACK_EMPTY_CHECK_REC(is_empty, mem, s);
#endif
        INC_OP;
        if (is_empty) {
#ifdef ONIG_DEBUG_MATCH
          fprintf(stderr, "EMPTY_CHECK_END_MEM_PUSH: skip  id:%d, s:%p\n",
                  (int )mem, s);
#endif
          if (is_empty == -1) goto fail;
          goto empty_check_found;
        }
        else {
          STACK_PUSH_EMPTY_CHECK_END(mem);
        }
      }
      JUMP_OUT;
#endif

    CASE_OP(JUMP)
      addr = p->jump.addr;
      p += addr;
      CHECK_INTERRUPT_JUMP_OUT;

    CASE_OP(PUSH)
      addr = p->push.addr;
      STACK_PUSH_ALT(p + addr, s, sprev);
      INC_OP;
      JUMP_OUT;

    CASE_OP(PUSH_SUPER)
      addr = p->push.addr;
      STACK_PUSH_SUPER_ALT(p + addr, s, sprev);
      INC_OP;
      JUMP_OUT;

    CASE_OP(POP_OUT)
      STACK_POP_ONE;
      /* for stop backtrack */
      /* CHECK_RETRY_LIMIT_IN_MATCH; */
      INC_OP;
      JUMP_OUT;

 #ifdef USE_OP_PUSH_OR_JUMP_EXACT
    CASE_OP(PUSH_OR_JUMP_EXACT1)
      {
        UChar c;

        addr = p->push_or_jump_exact1.addr;
        c    = p->push_or_jump_exact1.c;
        if (DATA_ENSURE_CHECK1 && c == *s) {
          STACK_PUSH_ALT(p + addr, s, sprev);
          INC_OP;
          JUMP_OUT;
        }
      }
      p += addr;
      JUMP_OUT;
#endif

    CASE_OP(PUSH_IF_PEEK_NEXT)
      {
        UChar c;

        addr = p->push_if_peek_next.addr;
        c    = p->push_if_peek_next.c;
        if (DATA_ENSURE_CHECK1 && c == *s) {
          STACK_PUSH_ALT(p + addr, s, sprev);
          INC_OP;
          JUMP_OUT;
        }
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(REPEAT)
      mem  = p->repeat.id;  /* mem: OP_REPEAT ID */
      addr = p->repeat.addr;

      STACK_PUSH_REPEAT_INC(mem, 0);
      if (reg->repeat_range[mem].lower == 0) {
        STACK_PUSH_ALT(p + addr, s, sprev);
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(REPEAT_NG)
      mem  = p->repeat.id;  /* mem: OP_REPEAT ID */
      addr = p->repeat.addr;

      STACK_PUSH_REPEAT_INC(mem, 0);
      if (reg->repeat_range[mem].lower == 0) {
        STACK_PUSH_ALT(p + 1, s, sprev);
        p += addr;
      }
      else
        INC_OP;
      JUMP_OUT;

    CASE_OP(REPEAT_INC)
      mem  = p->repeat_inc.id;  /* mem: OP_REPEAT ID */
      STACK_GET_REPEAT_COUNT(mem, n);
      n++;
      if (n >= reg->repeat_range[mem].upper) {
        /* end of repeat. Nothing to do. */
        INC_OP;
      }
      else if (n >= reg->repeat_range[mem].lower) {
        INC_OP;
        STACK_PUSH_ALT(p, s, sprev);
        p = reg->repeat_range[mem].u.pcode;
      }
      else {
        p = reg->repeat_range[mem].u.pcode;
      }
      STACK_PUSH_REPEAT_INC(mem, n);
      CHECK_INTERRUPT_JUMP_OUT;

    CASE_OP(REPEAT_INC_NG)
      mem = p->repeat_inc.id;  /* mem: OP_REPEAT ID */
      STACK_GET_REPEAT_COUNT(mem, n);
      n++;
      STACK_PUSH_REPEAT_INC(mem, n);
      if (n == reg->repeat_range[mem].upper) {
        INC_OP;
      }
      else {
        if (n >= reg->repeat_range[mem].lower) {
          STACK_PUSH_ALT(reg->repeat_range[mem].u.pcode, s, sprev);
          INC_OP;
        }
        else {
          p = reg->repeat_range[mem].u.pcode;
        }
      }
      CHECK_INTERRUPT_JUMP_OUT;

    CASE_OP(PREC_READ_START)
      STACK_PUSH_PREC_READ_START(s, sprev);
      INC_OP;
      JUMP_OUT;

    CASE_OP(PREC_READ_END)
      STACK_GET_PREC_READ_START(stkp);
      s     = stkp->u.state.pstr;
      sprev = stkp->u.state.pstr_prev;
      STACK_PUSH(STK_PREC_READ_END,0,0,0);
      INC_OP;
      JUMP_OUT;

    CASE_OP(PREC_READ_NOT_START)
      addr = p->prec_read_not_start.addr;
      STACK_PUSH_ALT_PREC_READ_NOT(p + addr, s, sprev);
      INC_OP;
      JUMP_OUT;

    CASE_OP(PREC_READ_NOT_END)
      STACK_POP_TIL_ALT_PREC_READ_NOT;
      goto fail;

    CASE_OP(ATOMIC_START)
      STACK_PUSH_TO_VOID_START;
      INC_OP;
      JUMP_OUT;

    CASE_OP(ATOMIC_END)
      STACK_EXEC_TO_VOID(stkp);
      INC_OP;
      JUMP_OUT;

    CASE_OP(LOOK_BEHIND)
      tlen = p->look_behind.len;
      s = (UChar* )ONIGENC_STEP_BACK(encode, str, s, (int )tlen);
      if (IS_NULL(s)) goto fail;
      sprev = (UChar* )onigenc_get_prev_char_head(encode, str, s);
      INC_OP;
      JUMP_OUT;

    CASE_OP(LOOK_BEHIND_NOT_START)
      addr = p->look_behind_not_start.addr;
      tlen = p->look_behind_not_start.len;
      q = (UChar* )ONIGENC_STEP_BACK(encode, str, s, (int )tlen);
      if (IS_NULL(q)) {
        /* too short case -> success. ex. /(?<!XXX)a/.match("a")
           If you want to change to fail, replace following line. */
        p += addr;
        /* goto fail; */
      }
      else {
        STACK_PUSH_ALT_LOOK_BEHIND_NOT(p + addr, s, sprev);
        s = q;
        sprev = (UChar* )onigenc_get_prev_char_head(encode, str, s);
        INC_OP;
      }
      JUMP_OUT;

    CASE_OP(LOOK_BEHIND_NOT_END)
      STACK_POP_TIL_ALT_LOOK_BEHIND_NOT;
      INC_OP;
      goto fail;

#ifdef USE_CALL
    CASE_OP(CALL)
      addr = p->call.addr;
      INC_OP; STACK_PUSH_CALL_FRAME(p);
      p = reg->ops + addr;
      JUMP_OUT;

    CASE_OP(RETURN)
      STACK_RETURN(p);
      STACK_PUSH_RETURN;
      JUMP_OUT;
#endif

    CASE_OP(PUSH_SAVE_VAL)
      {
        SaveType type;

        type = p->push_save_val.type;
        mem  = p->push_save_val.id; /* mem: save id */
        switch ((enum SaveType )type) {
        case SAVE_KEEP:
          STACK_PUSH_SAVE_VAL(mem, type, s);
          break;

        case SAVE_S:
          STACK_PUSH_SAVE_VAL_WITH_SPREV(mem, type, s);
          break;

        case SAVE_RIGHT_RANGE:
          STACK_PUSH_SAVE_VAL(mem, SAVE_RIGHT_RANGE, right_range);
          break;
        }
      }
      INC_OP;
      JUMP_OUT;

    CASE_OP(UPDATE_VAR)
      {
        UpdateVarType type;
        enum SaveType save_type;

        type = p->update_var.type;
        mem  = p->update_var.id; /* mem: save id */

        switch ((enum UpdateVarType )type) {
        case UPDATE_VAR_KEEP_FROM_STACK_LAST:
          STACK_GET_SAVE_VAL_TYPE_LAST(SAVE_KEEP, keep);
          break;
        case UPDATE_VAR_S_FROM_STACK:
          STACK_GET_SAVE_VAL_TYPE_LAST_ID_WITH_SPREV(SAVE_S, mem, s);
          break;
        case UPDATE_VAR_RIGHT_RANGE_FROM_S_STACK:
          save_type = SAVE_S;
          goto get_save_val_type_last_id;
          break;
        case UPDATE_VAR_RIGHT_RANGE_FROM_STACK:
          save_type = SAVE_RIGHT_RANGE;
        get_save_val_type_last_id:
          STACK_GET_SAVE_VAL_TYPE_LAST_ID(save_type, mem, right_range);
          break;
        case UPDATE_VAR_RIGHT_RANGE_INIT:
          INIT_RIGHT_RANGE;
          break;
        }
      }
      INC_OP;
      JUMP_OUT;

#ifdef USE_CALLOUT
    CASE_OP(CALLOUT_CONTENTS)
      of = ONIG_CALLOUT_OF_CONTENTS;
      mem = p->callout_contents.num;
      goto callout_common_entry;
      BREAK_OUT;

    CASE_OP(CALLOUT_NAME)
      {
        int call_result;
        int name_id;
        int in;
        CalloutListEntry* e;
        OnigCalloutFunc func;
        OnigCalloutArgs args;

        of  = ONIG_CALLOUT_OF_NAME;
        mem = p->callout_name.num;

      callout_common_entry:
        e = onig_reg_callout_list_at(reg, mem);
        in = e->in;
        if (of == ONIG_CALLOUT_OF_NAME) {
          name_id = p->callout_name.id;
          func = onig_get_callout_start_func(reg, mem);
        }
        else {
          name_id = ONIG_NON_NAME_ID;
          func = msa->mp->progress_callout_of_contents;
        }

        if (IS_NOT_NULL(func) && (in & ONIG_CALLOUT_IN_PROGRESS) != 0) {
          CALLOUT_BODY(func, ONIG_CALLOUT_IN_PROGRESS, name_id,
                       (int )mem, msa->mp->callout_user_data, args, call_result);
          switch (call_result) {
          case ONIG_CALLOUT_FAIL:
            goto fail;
            break;
          case ONIG_CALLOUT_SUCCESS:
            goto retraction_callout2;
            break;
          default: /* error code */
            if (call_result > 0) {
              call_result = ONIGERR_INVALID_ARGUMENT;
            }
            best_len = call_result;
            goto match_at_end;
            break;
          }
        }
        else {
        retraction_callout2:
          if ((in & ONIG_CALLOUT_IN_RETRACTION) != 0) {
            if (of == ONIG_CALLOUT_OF_NAME) {
              if (IS_NOT_NULL(func)) {
                STACK_PUSH_CALLOUT_NAME(name_id, mem, func);
              }
            }
            else {
              func = msa->mp->retraction_callout_of_contents;
              if (IS_NOT_NULL(func)) {
                STACK_PUSH_CALLOUT_CONTENTS(mem, func);
              }
            }
          }
        }
      }
      INC_OP;
      JUMP_OUT;
#endif

    CASE_OP(FINISH)
      goto match_at_end;

#ifdef ONIG_DEBUG_STATISTICS
    fail:
      SOP_OUT;
      goto fail2;
#endif
    CASE_OP(FAIL)
#ifdef ONIG_DEBUG_STATISTICS
    fail2:
#else
    fail:
#endif
      STACK_POP;
      p     = stk->u.state.pcode;
      s     = stk->u.state.pstr;
      sprev = stk->u.state.pstr_prev;
      CHECK_RETRY_LIMIT_IN_MATCH;
      JUMP_OUT;

    DEFAULT_OP
      MATCH_AT_ERROR_RETURN(ONIGERR_UNDEFINED_BYTECODE);

  } BYTECODE_INTERPRETER_END;

 match_at_end:
  STACK_SAVE;
  return best_len;
}


parse_branch(Node** top, PToken* tok, int term, UChar** src, UChar* end,
             ScanEnv* env, int group_head)
{
  int r;
  Node *node, **headp;

  *top = NULL;
  INC_PARSE_DEPTH(env->parse_depth);

  r = parse_exp(&node, tok, term, src, end, env, group_head);
  if (r < 0) {
    onig_node_free(node);
    return r;
  }

  if (r == TK_EOT || r == term || r == TK_ALT) {
    *top = node;
  }
  else {
    *top = node_new_list(node, NULL);
    if (IS_NULL(*top)) {
      onig_node_free(node);
      return ONIGERR_MEMORY;
    }

    headp = &(NODE_CDR(*top));
    while (r != TK_EOT && r != term && r != TK_ALT) {
      r = parse_exp(&node, tok, term, src, end, env, FALSE);
      if (r < 0) {
        onig_node_free(node);
        return r;
      }

      if (NODE_TYPE(node) == NODE_LIST) {
        *headp = node;
        while (IS_NOT_NULL(NODE_CDR(node))) node = NODE_CDR(node);
        headp = &(NODE_CDR(node));
      }
      else {
        *headp = node_new_list(node, NULL);
        headp = &(NODE_CDR(*headp));
      }
    }
  }

  DEC_PARSE_DEPTH(env->parse_depth);
  return r;
}


fetch_token_in_cc(PToken* tok, UChar** src, UChar* end, ScanEnv* env)
{
  int r;
  OnigCodePoint code;
  OnigCodePoint c, c2;
  OnigSyntaxType* syn = env->syntax;
  OnigEncoding enc = env->enc;
  UChar* prev;
  UChar* p = *src;
  PFETCH_READY;

  if (PEND) {
    tok->type = TK_EOT;
    return tok->type;
  }

  PFETCH(c);
  tok->type = TK_CHAR;
  tok->base = 0;
  tok->u.code = c;
  tok->escaped = 0;

  if (c == ']') {
    tok->type = TK_CC_CLOSE;
  }
  else if (c == '-') {
    tok->type = TK_CC_RANGE;
  }
  else if (c == MC_ESC(syn)) {
    if (! IS_SYNTAX_BV(syn, ONIG_SYN_BACKSLASH_ESCAPE_IN_CC))
      goto end;

    if (PEND) return ONIGERR_END_PATTERN_AT_ESCAPE;

    PFETCH(c);
    tok->escaped = 1;
    tok->u.code = c;
    switch (c) {
    case 'w':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_WORD;
      tok->u.prop.not   = 0;
      break;
    case 'W':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_WORD;
      tok->u.prop.not   = 1;
      break;
    case 'd':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT;
      tok->u.prop.not   = 0;
      break;
    case 'D':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT;
      tok->u.prop.not   = 1;
      break;
    case 's':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_SPACE;
      tok->u.prop.not   = 0;
      break;
    case 'S':
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_SPACE;
      tok->u.prop.not   = 1;
      break;
    case 'h':
      if (! IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_ESC_H_XDIGIT)) break;
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT;
      tok->u.prop.not   = 0;
      break;
    case 'H':
      if (! IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_ESC_H_XDIGIT)) break;
      tok->type = TK_CHAR_TYPE;
      tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT;
      tok->u.prop.not   = 1;
      break;

    case 'p':
    case 'P':
      if (PEND) break;

      c2 = PPEEK;
      if (c2 == '{' &&
          IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_ESC_P_BRACE_CHAR_PROPERTY)) {
        PINC;
        tok->type = TK_CHAR_PROPERTY;
        tok->u.prop.not = c == 'P';

        if (!PEND && IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_ESC_P_BRACE_CIRCUMFLEX_NOT)) {
          PFETCH(c2);
          if (c2 == '^') {
            tok->u.prop.not = tok->u.prop.not == 0;
          }
          else
            PUNFETCH;
        }
      }
      break;

    case 'o':
      if (PEND) break;

      prev = p;
      if (PPEEK_IS('{') && IS_SYNTAX_OP(syn, ONIG_SYN_OP_ESC_O_BRACE_OCTAL)) {
        PINC;
        r = scan_octal_number(&p, end, 0, 11, enc, &code);
        if (r < 0) return r;
        if (!PEND) {
          c2 = PPEEK;
          if (IS_CODE_DIGIT_ASCII(enc, c2))
            return ONIGERR_TOO_LONG_WIDE_CHAR_VALUE;
        }

        if (p > prev + enclen(enc, prev) && !PEND && (PPEEK_IS('}'))) {
          PINC;
          tok->type   = TK_CODE_POINT;
          tok->base   = 8;
          tok->u.code = code;
        }
        else {
          /* can't read nothing or invalid format */
          p = prev;
        }
      }
      break;

    case 'x':
      if (PEND) break;

      prev = p;
      if (PPEEK_IS('{') && IS_SYNTAX_OP(syn, ONIG_SYN_OP_ESC_X_BRACE_HEX8)) {
        PINC;
        r = scan_hexadecimal_number(&p, end, 0, 8, enc, &code);
        if (r < 0) return r;
        if (!PEND) {
          c2 = PPEEK;
          if (IS_CODE_XDIGIT_ASCII(enc, c2))
            return ONIGERR_TOO_LONG_WIDE_CHAR_VALUE;
        }

        if (p > prev + enclen(enc, prev) && !PEND && (PPEEK_IS('}'))) {
          PINC;
          tok->type   = TK_CODE_POINT;
          tok->base   = 16;
          tok->u.code = code;
        }
        else {
          /* can't read nothing or invalid format */
          p = prev;
        }
      }
      else if (IS_SYNTAX_OP(syn, ONIG_SYN_OP_ESC_X_HEX2)) {
        r = scan_hexadecimal_number(&p, end, 0, 2, enc, &code);
        if (r < 0) return r;
        if (p == prev) {  /* can't read nothing. */
          code = 0; /* but, it's not error */
        }
        tok->type = TK_CRUDE_BYTE;
        tok->base = 16;
        tok->u.byte = (UChar )code;
      }
      break;

    case 'u':
      if (PEND) break;

      prev = p;
      if (IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_ESC_U_HEX4)) {
        r = scan_hexadecimal_number(&p, end, 4, 4, enc, &code);
        if (r < 0) return r;
        if (p == prev) {  /* can't read nothing. */
          code = 0; /* but, it's not error */
        }
        tok->type   = TK_CODE_POINT;
        tok->base   = 16;
        tok->u.code = code;
      }
      break;

    case '0':
    case '1': case '2': case '3': case '4': case '5': case '6': case '7':
      if (IS_SYNTAX_OP(syn, ONIG_SYN_OP_ESC_OCTAL3)) {
        PUNFETCH;
        prev = p;
        r = scan_octal_number(&p, end, 0, 3, enc, &code);
        if (r < 0) return r;
        if (code >= 256) return ONIGERR_TOO_BIG_NUMBER;
        if (p == prev) {  /* can't read nothing. */
          code = 0; /* but, it's not error */
        }
        tok->type = TK_CRUDE_BYTE;
        tok->base = 8;
        tok->u.byte = (UChar )code;
      }
      break;

    default:
      PUNFETCH;
      r = fetch_escaped_value(&p, end, env, &c2);
      if (r < 0) return r;
      if (tok->u.code != c2) {
        tok->u.code = c2;
        tok->type   = TK_CODE_POINT;
      }
      break;
    }
  }
  else if (c == '[') {
    if (IS_SYNTAX_OP(syn, ONIG_SYN_OP_POSIX_BRACKET) && (PPEEK_IS(':'))) {
      OnigCodePoint send[] = { (OnigCodePoint )':', (OnigCodePoint )']' };
      tok->backp = p; /* point at '[' is read */
      PINC;
      if (str_exist_check_with_esc(send, 2, p, end,
                                   (OnigCodePoint )']', enc, syn)) {
        tok->type = TK_CC_POSIX_BRACKET_OPEN;
      }
      else {
        PUNFETCH;
        goto cc_in_cc;
      }
    }
    else {
    cc_in_cc:
      if (IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_CCLASS_SET_OP)) {
        tok->type = TK_CC_OPEN_CC;
      }
      else {
        CC_ESC_WARN(env, (UChar* )"[");
      }
    }
  }
  else if (c == '&') {
    if (IS_SYNTAX_OP2(syn, ONIG_SYN_OP2_CCLASS_SET_OP) &&
        !PEND && (PPEEK_IS('&'))) {
      PINC;
      tok->type = TK_CC_AND;
    }
  }

 end:
  *src = p;
  return tok->type;
}


node_new_anychar(void)
{
  Node* node = node_new_ctype(CTYPE_ANYCHAR, FALSE, ONIG_OPTION_NONE);
  return node;
}


gb18030_left_adjust_char_head(const UChar* start, const UChar* s)
{
  const UChar *p;
  enum state state = S_START;

  DEBUG_GB18030(("----------------\n"));
  for (p = s; p >= start; p--) {
    DEBUG_GB18030(("state %d --(%02x)-->\n", state, *p));
    switch (state) {
    case S_START:
      switch (GB18030_MAP[*p]) {
      case C1:
        return (UChar *)s;
      case C2:
        state = S_one_C2; /* C2 */
        break;
      case C4:
        state = S_one_C4; /* C4 */
        break;
      case CM:
        state = S_one_CM; /* CM */
        break;
      }
      break;
    case S_one_C2: /* C2 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)s;
      case CM:
        state = S_odd_CM_one_CX; /* CM C2 */
        break;
      }
      break;
    case S_one_C4: /* C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)s;
      case CM:
        state = S_one_CMC4;
        break;
      }
      break;
    case S_one_CM: /* CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)s;
      case C4:
        state = S_odd_C4CM;
        break;
      case CM:
        state = S_odd_CM_one_CX; /* CM CM */
        break;
      }
      break;

    case S_odd_CM_one_CX: /* CM C2 */ /* CM CM */ /* CM CM CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 1);
      case CM:
        state = S_even_CM_one_CX;
        break;
      }
      break;
    case S_even_CM_one_CX: /* CM CM C2 */ /* CM CM CM */ /* CM CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)s;
      case CM:
        state = S_odd_CM_one_CX;
        break;
      }
      break;

    case S_one_CMC4: /* CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)(s - 1);
      case C4:
        state = S_one_C4_odd_CMC4; /* C4 CM C4 */
        break;
      case CM:
        state = S_even_CM_one_CX; /* CM CM C4 */
        break;
      }
      break;
    case S_odd_CMC4: /* CM C4 CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)(s - 1);
      case C4:
        state = S_one_C4_odd_CMC4;
        break;
      case CM:
        state = S_odd_CM_odd_CMC4;
        break;
      }
      break;
    case S_one_C4_odd_CMC4: /* C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 1);
      case CM:
        state = S_even_CMC4; /* CM C4 CM C4 */
        break;
      }
      break;
    case S_even_CMC4: /* CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)(s - 3);
      case C4:
        state = S_one_C4_even_CMC4;
        break;
      case CM:
        state = S_odd_CM_even_CMC4;
        break;
      }
      break;
    case S_one_C4_even_CMC4: /* C4 CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 3);
      case CM:
        state = S_odd_CMC4;
        break;
      }
      break;

    case S_odd_CM_odd_CMC4: /* CM CM C4 CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 3);
      case CM:
        state = S_even_CM_odd_CMC4;
        break;
      }
      break;
    case S_even_CM_odd_CMC4: /* CM CM CM C4 CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 1);
      case CM:
        state = S_odd_CM_odd_CMC4;
        break;
      }
      break;

    case S_odd_CM_even_CMC4: /* CM CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 1);
      case CM:
        state = S_even_CM_even_CMC4;
        break;
      }
      break;
    case S_even_CM_even_CMC4: /* CM CM CM C4 CM C4 */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 3);
      case CM:
        state = S_odd_CM_even_CMC4;
        break;
      }
      break;

    case S_odd_C4CM: /* C4 CM */  /* C4 CM C4 CM C4 CM*/
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)s;
      case CM:
        state = S_one_CM_odd_C4CM; /* CM C4 CM */
        break;
      }
      break;
    case S_one_CM_odd_C4CM: /* CM C4 CM */ /* CM C4 CM C4 CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)(s - 2); /* |CM C4 CM */
      case C4:
        state = S_even_C4CM;
        break;
      case CM:
        state = S_even_CM_odd_C4CM;
        break;
      }
      break;
    case S_even_C4CM: /* C4 CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 2);  /* C4|CM C4 CM */
      case CM:
        state = S_one_CM_even_C4CM;
        break;
      }
      break;
    case S_one_CM_even_C4CM: /* CM C4 CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
        return (UChar *)(s - 0);  /*|CM C4 CM C4|CM */
      case C4:
        state = S_odd_C4CM;
        break;
      case CM:
        state = S_even_CM_even_C4CM;
        break;
      }
      break;

    case S_even_CM_odd_C4CM: /* CM CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 0); /* |CM CM|C4|CM */
      case CM:
        state = S_odd_CM_odd_C4CM;
        break;
      }
      break;
    case S_odd_CM_odd_C4CM: /* CM CM CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 2); /* |CM CM|CM C4 CM */
      case CM:
        state = S_even_CM_odd_C4CM;
        break;
      }
      break;

    case S_even_CM_even_C4CM: /* CM CM C4 CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 2); /* |CM CM|C4|CM C4 CM */
      case CM:
        state = S_odd_CM_even_C4CM;
        break;
      }
      break;
    case S_odd_CM_even_C4CM: /* CM CM CM C4 CM C4 CM */
      switch (GB18030_MAP[*p]) {
      case C1:
      case C2:
      case C4:
        return (UChar *)(s - 0);  /* |CM CM|CM C4 CM C4|CM */
      case CM:
        state = S_even_CM_even_C4CM;
        break;
      }
      break;
    }
  }

  DEBUG_GB18030(("state %d\n", state));
  switch (state) {
  case S_START:             return (UChar *)(s - 0);
  case S_one_C2:            return (UChar *)(s - 0);
  case S_one_C4:            return (UChar *)(s - 0);
  case S_one_CM:            return (UChar *)(s - 0);

  case S_odd_CM_one_CX:     return (UChar *)(s - 1);
  case S_even_CM_one_CX:    return (UChar *)(s - 0);

  case S_one_CMC4:          return (UChar *)(s - 1);
  case S_odd_CMC4:          return (UChar *)(s - 1);
  case S_one_C4_odd_CMC4:   return (UChar *)(s - 1);
  case S_even_CMC4:         return (UChar *)(s - 3);
  case S_one_C4_even_CMC4:  return (UChar *)(s - 3);

  case S_odd_CM_odd_CMC4:   return (UChar *)(s - 3);
  case S_even_CM_odd_CMC4:  return (UChar *)(s - 1);

  case S_odd_CM_even_CMC4:  return (UChar *)(s - 1);
  case S_even_CM_even_CMC4: return (UChar *)(s - 3);

  case S_odd_C4CM:          return (UChar *)(s - 0);
  case S_one_CM_odd_C4CM:   return (UChar *)(s - 2);
  case S_even_C4CM:         return (UChar *)(s - 2);
  case S_one_CM_even_C4CM:  return (UChar *)(s - 0);

  case S_even_CM_odd_C4CM:  return (UChar *)(s - 0);
  case S_odd_CM_odd_C4CM:   return (UChar *)(s - 2);
  case S_even_CM_even_C4CM: return (UChar *)(s - 2);
  case S_odd_CM_even_C4CM:  return (UChar *)(s - 0);
  }

  return (UChar* )s;  /* never come here. (escape warning) */
}


parse_exp(Node** np, OnigToken* tok, int term,
	  UChar** src, UChar* end, ScanEnv* env)
{
  int r, len, group = 0;
  Node* qn;
  Node** targetp;

  *np = NULL;
  if (tok->type == (enum TokenSyms )term)
    goto end_of_token;

  switch (tok->type) {
  case TK_ALT:
  case TK_EOT:
  end_of_token:
    *np = node_new_empty();
    return tok->type;
    break;

  case TK_SUBEXP_OPEN:
    r = parse_enclose(np, tok, TK_SUBEXP_CLOSE, src, end, env);
    if (r < 0) return r;
    if (r == 1) group = 1;
    else if (r == 2) { /* option only */
      Node* target;
      OnigOptionType prev = env->option;

      env->option = NENCLOSE(*np)->option;
      r = fetch_token(tok, src, end, env);
      if (r < 0) return r;
      r = parse_subexp(&target, tok, term, src, end, env);
      env->option = prev;
      if (r < 0) {
	onig_node_free(target);
	return r;
      }
      NENCLOSE(*np)->target = target;
      return tok->type;
    }
    break;

  case TK_SUBEXP_CLOSE:
    if (! IS_SYNTAX_BV(env->syntax, ONIG_SYN_ALLOW_UNMATCHED_CLOSE_SUBEXP))
      return ONIGERR_UNMATCHED_CLOSE_PARENTHESIS;

    if (tok->escaped) goto tk_raw_byte;
    else goto tk_byte;
    break;

  case TK_LINEBREAK:
    r = node_linebreak(np, env);
    if (r < 0) return r;
    break;

  case TK_EXTENDED_GRAPHEME_CLUSTER:
    r = node_extended_grapheme_cluster(np, env);
    if (r < 0) return r;
    break;

  case TK_KEEP:
    *np = onig_node_new_anchor(ANCHOR_KEEP);
    CHECK_NULL_RETURN_MEMERR(*np);
    break;

  case TK_STRING:
  tk_byte:
    {
      *np = node_new_str(tok->backp, *src);
      CHECK_NULL_RETURN_MEMERR(*np);

    string_loop:
      while (1) {
	r = fetch_token(tok, src, end, env);
	if (r < 0) return r;
	if (r == TK_STRING) {
	  r = onig_node_str_cat(*np, tok->backp, *src);
	}
#ifndef NUMBERED_CHAR_IS_NOT_CASE_AMBIG
	else if (r == TK_CODE_POINT) {
	  r = node_str_cat_codepoint(*np, env->enc, tok->u.code);
	}
#endif
	else {
	  break;
	}
	if (r < 0) return r;
      }

    string_end:
      targetp = np;
      goto repeat;
    }
    break;

  case TK_RAW_BYTE:
  tk_raw_byte:
    {
      *np = node_new_str_raw_char((UChar )tok->u.c);
      CHECK_NULL_RETURN_MEMERR(*np);
      len = 1;
      while (1) {
	if (len >= ONIGENC_MBC_MINLEN(env->enc)) {
	  if (len == enclen(env->enc, NSTR(*np)->s, NSTR(*np)->end)) {
	    r = fetch_token(tok, src, end, env);
	    NSTRING_CLEAR_RAW(*np);
	    goto string_end;
	  }
	}

	r = fetch_token(tok, src, end, env);
	if (r < 0) return r;
	if (r != TK_RAW_BYTE) {
	  /* Don't use this, it is wrong for little endian encodings. */
#ifdef USE_PAD_TO_SHORT_BYTE_CHAR
	  int rem;
	  if (len < ONIGENC_MBC_MINLEN(env->enc)) {
	    rem = ONIGENC_MBC_MINLEN(env->enc) - len;
	    (void )node_str_head_pad(NSTR(*np), rem, (UChar )0);
	    if (len + rem == enclen(env->enc, NSTR(*np)->s)) {
	      NSTRING_CLEAR_RAW(*np);
	      goto string_end;
	    }
	  }
#endif
	  return ONIGERR_TOO_SHORT_MULTI_BYTE_STRING;
	}

	r = node_str_cat_char(*np, (UChar )tok->u.c);
	if (r < 0) return r;

	len++;
      }
    }
    break;

  case TK_CODE_POINT:
    {
      *np = node_new_empty();
      CHECK_NULL_RETURN_MEMERR(*np);
      r = node_str_cat_codepoint(*np, env->enc, tok->u.code);
      if (r != 0) return r;
#ifdef NUMBERED_CHAR_IS_NOT_CASE_AMBIG
      NSTRING_SET_RAW(*np);
#else
      goto string_loop;
#endif
    }
    break;

  case TK_QUOTE_OPEN:
    {
      OnigCodePoint end_op[2];
      UChar *qstart, *qend, *nextp;

      end_op[0] = (OnigCodePoint )MC_ESC(env->syntax);
      end_op[1] = (OnigCodePoint )'E';
      qstart = *src;
      qend = find_str_position(end_op, 2, qstart, end, &nextp, env->enc);
      if (IS_NULL(qend)) {
	nextp = qend = end;
      }
      *np = node_new_str(qstart, qend);
      CHECK_NULL_RETURN_MEMERR(*np);
      *src = nextp;
    }
    break;

  case TK_CHAR_TYPE:
    {
      switch (tok->u.prop.ctype) {
      case ONIGENC_CTYPE_WORD:
	*np = node_new_ctype(tok->u.prop.ctype, tok->u.prop.not,
			     IS_ASCII_RANGE(env->option));
	CHECK_NULL_RETURN_MEMERR(*np);
	break;

      case ONIGENC_CTYPE_SPACE:
      case ONIGENC_CTYPE_DIGIT:
      case ONIGENC_CTYPE_XDIGIT:
	{
	  CClassNode* cc;

#ifdef USE_SHARED_CCLASS_TABLE
	  const OnigCodePoint *mbr;
	  OnigCodePoint sb_out;

	  r = ONIGENC_GET_CTYPE_CODE_RANGE(env->enc, tok->u.prop.ctype,
	      &sb_out, &mbr);
	  if (r == 0 &&
	      ! IS_ASCII_RANGE(env->option) &&
	      ONIGENC_CODE_RANGE_NUM(mbr)
	      >= THRESHOLD_RANGE_NUM_FOR_SHARE_CCLASS) {
	    type_cclass_key  key;
	    type_cclass_key* new_key;

	    key.enc  = env->enc;
	    key.not  = tok->u.prop.not;
	    key.type = tok->u.prop.ctype;

	    THREAD_ATOMIC_START;

	    if (IS_NULL(OnigTypeCClassTable)) {
	      OnigTypeCClassTable
		= onig_st_init_table_with_size(&type_type_cclass_hash, 10);
	      if (IS_NULL(OnigTypeCClassTable)) {
		THREAD_ATOMIC_END;
		return ONIGERR_MEMORY;
	      }
	    }
	    else {
	      if (onig_st_lookup(OnigTypeCClassTable, (st_data_t )&key,
		    (st_data_t* )np)) {
		THREAD_ATOMIC_END;
		break;
	      }
	    }

	    *np = node_new_cclass_by_codepoint_range(tok->u.prop.not,
		sb_out, mbr);
	    if (IS_NULL(*np)) {
	      THREAD_ATOMIC_END;
	      return ONIGERR_MEMORY;
	    }

	    cc = NCCLASS(*np);
	    NCCLASS_SET_SHARE(cc);
	    new_key = (type_cclass_key* )xmalloc(sizeof(type_cclass_key));
	    xmemcpy(new_key, &key, sizeof(type_cclass_key));
	    onig_st_add_direct(OnigTypeCClassTable, (st_data_t )new_key,
		(st_data_t )*np);

	    THREAD_ATOMIC_END;
	  }
	  else {
#endif
	    *np = node_new_cclass();
	    CHECK_NULL_RETURN_MEMERR(*np);
	    cc = NCCLASS(*np);
	    r = add_ctype_to_cc(cc, tok->u.prop.ctype, 0,
		IS_ASCII_RANGE(env->option), env);
	    if (r != 0) return r;
	    if (tok->u.prop.not != 0) NCCLASS_SET_NOT(cc);
#ifdef USE_SHARED_CCLASS_TABLE
	  }
#endif
	}
	break;

      default:
	return ONIGERR_PARSER_BUG;
	break;
      }
    }
    break;

  case TK_CHAR_PROPERTY:
    r = parse_char_property(np, tok, src, end, env);
    if (r != 0) return r;
    break;

  case TK_CC_OPEN:
    {
      Node *asc_node;
      CClassNode* cc;
      OnigCodePoint code;

      r = parse_char_class(np, &asc_node, tok, src, end, env);
      if (r != 0) {
	onig_node_free(asc_node);
	return r;
      }

      cc = NCCLASS(*np);
      if (is_onechar_cclass(cc, &code)) {
	onig_node_free(*np);
	onig_node_free(asc_node);
	*np = node_new_empty();
	CHECK_NULL_RETURN_MEMERR(*np);
	r = node_str_cat_codepoint(*np, env->enc, code);
	if (r != 0) return r;
	goto string_loop;
      }
      if (IS_IGNORECASE(env->option)) {
	r = cclass_case_fold(np, cc, NCCLASS(asc_node), env);
	if (r != 0) {
	  onig_node_free(asc_node);
	  return r;
	}
      }
      onig_node_free(asc_node);
    }
    break;

  case TK_ANYCHAR:
    *np = node_new_anychar();
    CHECK_NULL_RETURN_MEMERR(*np);
    break;

  case TK_ANYCHAR_ANYTIME:
    *np = node_new_anychar();
    CHECK_NULL_RETURN_MEMERR(*np);
    qn = node_new_quantifier(0, REPEAT_INFINITE, 0);
    CHECK_NULL_RETURN_MEMERR(qn);
    NQTFR(qn)->target = *np;
    *np = qn;
    break;

  case TK_BACKREF:
    len = tok->u.backref.num;
    *np = node_new_backref(len,
		   (len > 1 ? tok->u.backref.refs : &(tok->u.backref.ref1)),
			   tok->u.backref.by_name,
#ifdef USE_BACKREF_WITH_LEVEL
			   tok->u.backref.exist_level,
			   tok->u.backref.level,
#endif
			   env);
    CHECK_NULL_RETURN_MEMERR(*np);
    break;

#ifdef USE_SUBEXP_CALL
  case TK_CALL:
    {
      int gnum = tok->u.call.gnum;

      if (gnum < 0 || tok->u.call.rel != 0) {
	if (gnum > 0) gnum--;
	gnum = BACKREF_REL_TO_ABS(gnum, env);
	if (gnum <= 0)
	  return ONIGERR_INVALID_BACKREF;
      }
      *np = node_new_call(tok->u.call.name, tok->u.call.name_end, gnum);
      CHECK_NULL_RETURN_MEMERR(*np);
      env->num_call++;
    }
    break;
#endif

  case TK_ANCHOR:
    *np = onig_node_new_anchor(tok->u.anchor.subtype);
    CHECK_NULL_RETURN_MEMERR(*np);
    NANCHOR(*np)->ascii_range = tok->u.anchor.ascii_range;
    break;

  case TK_OP_REPEAT:
  case TK_INTERVAL:
    if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_CONTEXT_INDEP_REPEAT_OPS)) {
      if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_CONTEXT_INVALID_REPEAT_OPS))
	return ONIGERR_TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED;
      else
	*np = node_new_empty();
    }
    else {
      goto tk_byte;
    }
    break;

  default:
    return ONIGERR_PARSER_BUG;
    break;
  }

  {
    targetp = np;

  re_entry:
    r = fetch_token(tok, src, end, env);
    if (r < 0) return r;

  repeat:
    if (r == TK_OP_REPEAT || r == TK_INTERVAL) {
      if (is_invalid_quantifier_target(*targetp))
	return ONIGERR_TARGET_OF_REPEAT_OPERATOR_INVALID;

      qn = node_new_quantifier(tok->u.repeat.lower, tok->u.repeat.upper,
			       (r == TK_INTERVAL ? 1 : 0));
      CHECK_NULL_RETURN_MEMERR(qn);
      NQTFR(qn)->greedy = tok->u.repeat.greedy;
      r = set_quantifier(qn, *targetp, group, env);
      if (r < 0) {
	onig_node_free(qn);
	return r;
      }

      if (tok->u.repeat.possessive != 0) {
	Node* en;
	en = node_new_enclose(ENCLOSE_STOP_BACKTRACK);
	if (IS_NULL(en)) {
	  onig_node_free(qn);
	  return ONIGERR_MEMORY;
	}
	NENCLOSE(en)->target = qn;
	qn = en;
      }

      if (r == 0) {
	*targetp = qn;
      }
      else if (r == 1) {
	onig_node_free(qn);
      }
      else if (r == 2) { /* split case: /abc+/ */
	Node *tmp;

	*targetp = node_new_list(*targetp, NULL);
	if (IS_NULL(*targetp)) {
	  onig_node_free(qn);
	  return ONIGERR_MEMORY;
	}
	tmp = NCDR(*targetp) = node_new_list(qn, NULL);
	if (IS_NULL(tmp)) {
	  onig_node_free(qn);
	  return ONIGERR_MEMORY;
	}
	targetp = &(NCAR(tmp));
      }
      goto re_entry;
    }
  }

  return r;
}


node_new_cclass_by_codepoint_range(int not, OnigCodePoint sb_out,
				   const OnigCodePoint ranges[])
{
  int n, i;
  CClassNode* cc;
  OnigCodePoint j;

  Node* node = node_new_cclass_locked();
  CHECK_NULL_RETURN(node);

  cc = NCCLASS(node);
  if (not != 0) NCCLASS_SET_NOT(cc);

  BITSET_CLEAR(cc->bs);
  if (sb_out > 0 && IS_NOT_NULL(ranges)) {
    n = ONIGENC_CODE_RANGE_NUM(ranges);
    for (i = 0; i < n; i++) {
      for (j = ONIGENC_CODE_RANGE_FROM(ranges, i);
	  j <= (OnigCodePoint )ONIGENC_CODE_RANGE_TO(ranges, i); j++) {
	if (j >= sb_out) goto sb_end;

	BITSET_SET_BIT(cc->bs, j);
      }
    }
  }

 sb_end:
  if (IS_NULL(ranges)) {
  is_null:
    cc->mbuf = NULL;
  }
  else {
    BBuf* bbuf;

    n = ONIGENC_CODE_RANGE_NUM(ranges);
    if (n == 0) goto is_null;

    bbuf = (BBuf* )xmalloc(sizeof(BBuf));
    CHECK_NULL_RETURN(bbuf);
    bbuf->alloc = n + 1;
    bbuf->used  = n + 1;
    bbuf->p     = (UChar* )((void* )ranges);

    cc->mbuf = bbuf;
  }

  return node;
}


static void sm501_palette_write(void *opaque, hwaddr addr,
                                uint32_t value)
{
    SM501State *s = (SM501State *)opaque;
    SM501_DPRINTF("sm501 palette write addr=%x, val=%x\n",
                  (int)addr, value);

    /* TODO : consider BYTE/WORD access */
    /* TODO : consider endian */

    assert(range_covers_byte(0, 0x400 * 3, addr));
    *(uint32_t *)&s->dc_palette[addr] = value;
    s->do_full_update = true;
}


static void sm501_init(SM501State *s, DeviceState *dev,
                       uint32_t local_mem_bytes)
{
    s->local_mem_size_index = get_local_mem_size_index(local_mem_bytes);
    SM501_DPRINTF("sm501 local mem size=%x. index=%d\n", get_local_mem_size(s),
                  s->local_mem_size_index);

    /* local memory */
    memory_region_init_ram(&s->local_mem_region, OBJECT(dev), "sm501.local",
                           get_local_mem_size(s), &error_fatal);
    memory_region_set_log(&s->local_mem_region, true, DIRTY_MEMORY_VGA);
    s->local_mem = memory_region_get_ram_ptr(&s->local_mem_region);

    /* i2c */
    s->i2c_bus = i2c_init_bus(dev, "sm501.i2c");
    /* ddc */
    I2CDDCState *ddc = I2CDDC(qdev_create(BUS(s->i2c_bus), TYPE_I2CDDC));
    i2c_set_slave_address(I2C_SLAVE(ddc), 0x50);

    /* mmio */
    memory_region_init(&s->mmio_region, OBJECT(dev), "sm501.mmio", MMIO_SIZE);
    memory_region_init_io(&s->system_config_region, OBJECT(dev),
                          &sm501_system_config_ops, s,
                          "sm501-system-config", 0x6c);
    memory_region_add_subregion(&s->mmio_region, SM501_SYS_CONFIG,
                                &s->system_config_region);
    memory_region_init_io(&s->i2c_region, OBJECT(dev), &sm501_i2c_ops, s,
                          "sm501-i2c", 0x14);
    memory_region_add_subregion(&s->mmio_region, SM501_I2C, &s->i2c_region);
    memory_region_init_io(&s->disp_ctrl_region, OBJECT(dev),
                          &sm501_disp_ctrl_ops, s,
                          "sm501-disp-ctrl", 0x1000);
    memory_region_add_subregion(&s->mmio_region, SM501_DC,
                                &s->disp_ctrl_region);
    memory_region_init_io(&s->twoD_engine_region, OBJECT(dev),
                          &sm501_2d_engine_ops, s,
                          "sm501-2d-engine", 0x54);
    memory_region_add_subregion(&s->mmio_region, SM501_2D_ENGINE,
                                &s->twoD_engine_region);

    /* create qemu graphic console */
    s->con = graphic_console_init(dev, 0, &sm501_ops, s);
}


static inline int get_width(SM501State *s, int crt)
{
    int width = crt ? s->dc_crt_h_total : s->dc_panel_h_total;
    return (width & 0x00000FFF) + 1;
}


static void sm501_realize_sysbus(DeviceState *dev, Error **errp)
{
    SM501SysBusState *s = SYSBUS_SM501(dev);
    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
    DeviceState *usb_dev;
    MemoryRegion *mr;

    sm501_init(&s->state, dev, s->vram_size);
    if (get_local_mem_size(&s->state) != s->vram_size) {
        error_setg(errp, "Invalid VRAM size, nearest valid size is %" PRIu32,
                   get_local_mem_size(&s->state));
        return;
    }
    sysbus_init_mmio(sbd, &s->state.local_mem_region);
    sysbus_init_mmio(sbd, &s->state.mmio_region);

    /* bridge to usb host emulation module */
    usb_dev = qdev_create(NULL, "sysbus-ohci");
    qdev_prop_set_uint32(usb_dev, "num-ports", 2);
    qdev_prop_set_uint64(usb_dev, "dma-offset", s->base);
    qdev_init_nofail(usb_dev);
    memory_region_add_subregion(&s->state.mmio_region, SM501_USB_HOST,
                       sysbus_mmio_get_region(SYS_BUS_DEVICE(usb_dev), 0));
    sysbus_pass_irq(sbd, SYS_BUS_DEVICE(usb_dev));

    /* bridge to serial emulation module */
    qdev_init_nofail(DEVICE(&s->serial));
    mr = sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->serial), 0);
    memory_region_add_subregion(&s->state.mmio_region, SM501_UART0, mr);
    /* TODO : chain irq to IRL */
}


static void sm501_realize_pci(PCIDevice *dev, Error **errp)
{
    SM501PCIState *s = PCI_SM501(dev);

    sm501_init(&s->state, DEVICE(dev), s->vram_size);
    if (get_local_mem_size(&s->state) != s->vram_size) {
        error_setg(errp, "Invalid VRAM size, nearest valid size is %" PRIu32,
                   get_local_mem_size(&s->state));
        return;
    }
    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
                     &s->state.local_mem_region);
    pci_register_bar(dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
                     &s->state.mmio_region);
}


static PHP_INI_MH(OnUpdateName) /* {{{ */
{
	SESSION_CHECK_ACTIVE_STATE;
	SESSION_CHECK_OUTPUT_STATE;

	/* Numeric session.name won't work at all */
	if ((!ZSTR_LEN(new_value) || is_numeric_string(ZSTR_VAL(new_value), ZSTR_LEN(new_value), NULL, NULL, 0))) {
		int err_type;

		if (stage == ZEND_INI_STAGE_RUNTIME || stage == ZEND_INI_STAGE_ACTIVATE || stage == ZEND_INI_STAGE_STARTUP) {
			err_type = E_WARNING;
		} else {
			err_type = E_ERROR;
		}

		/* Do not output error when restoring ini options. */
		if (stage != ZEND_INI_STAGE_DEACTIVATE) {
			php_error_docref(NULL, err_type, "session.name cannot be a numeric or empty '%s'", ZSTR_VAL(new_value));
		}
		return FAILURE;
	}

	return OnUpdateStringUnempty(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage);
}


static int php_session_initialize(void) /* {{{ */
{
	zend_string *val = NULL;

	PS(session_status) = php_session_active;

	if (!PS(mod)) {
		PS(session_status) = php_session_disabled;
		php_error_docref(NULL, E_WARNING, "No storage module chosen - failed to initialize session");
		return FAILURE;
	}

	/* Open session handler first */
	if (PS(mod)->s_open(&PS(mod_data), PS(save_path), PS(session_name)) == FAILURE
		/* || PS(mod_data) == NULL */ /* FIXME: open must set valid PS(mod_data) with success */
	) {
		php_session_abort();
		php_error_docref(NULL, E_WARNING, "Failed to initialize storage module: %s (path: %s)", PS(mod)->s_name, PS(save_path));
		return FAILURE;
	}

	/* If there is no ID, use session module to create one */
	if (!PS(id) || !ZSTR_VAL(PS(id))[0]) {
		if (PS(id)) {
			zend_string_release(PS(id));
		}
		PS(id) = PS(mod)->s_create_sid(&PS(mod_data));
		if (!PS(id)) {
			php_session_abort();
			zend_throw_error(NULL, "Failed to create session ID: %s (path: %s)", PS(mod)->s_name, PS(save_path));
			return FAILURE;
		}
		if (PS(use_cookies)) {
			PS(send_cookie) = 1;
		}
	} else if (PS(use_strict_mode) && PS(mod)->s_validate_sid &&
		PS(mod)->s_validate_sid(&PS(mod_data), PS(id)) == FAILURE) {
		if (PS(id)) {
			zend_string_release(PS(id));
		}
		PS(id) = PS(mod)->s_create_sid(&PS(mod_data));
		if (!PS(id)) {
			PS(id) = php_session_create_id(NULL);
		}
		if (PS(use_cookies)) {
			PS(send_cookie) = 1;
		}
	}

	if (php_session_reset_id() == FAILURE) {
		php_session_abort();
		return FAILURE;
	}

	/* Read data */
	php_session_track_init();
	if (PS(mod)->s_read(&PS(mod_data), PS(id), &val, PS(gc_maxlifetime)) == FAILURE) {
		php_session_abort();
		/* FYI: Some broken save handlers return FAILURE for non-existent session ID, this is incorrect */
		php_error_docref(NULL, E_WARNING, "Failed to read session data: %s (path: %s)", PS(mod)->s_name, PS(save_path));
		return FAILURE;
	}

	/* GC must be done after read */
	php_session_gc(0);

	if (PS(session_vars)) {
		zend_string_release(PS(session_vars));
		PS(session_vars) = NULL;
	}
	if (val) {
		if (PS(lazy_write)) {
			PS(session_vars) = zend_string_copy(val);
		}
		php_session_decode(val);
		zend_string_release(val);
	}
	return SUCCESS;
}


static PHP_INI_MH(OnUpdateSerializer) /* {{{ */
{
	const ps_serializer *tmp;

	SESSION_CHECK_ACTIVE_STATE;
	SESSION_CHECK_OUTPUT_STATE;

	tmp = _php_find_ps_serializer(ZSTR_VAL(new_value));

	if (PG(modules_activated) && !tmp) {
		int err_type;

		if (stage == ZEND_INI_STAGE_RUNTIME) {
			err_type = E_WARNING;
		} else {
			err_type = E_ERROR;
		}

		/* Do not output error when restoring ini options. */
		if (stage != ZEND_INI_STAGE_DEACTIVATE) {
			php_error_docref(NULL, err_type, "Cannot find serialization handler '%s'", ZSTR_VAL(new_value));
		}
		return FAILURE;
	}
	PS(serializer) = tmp;

	return SUCCESS;
}


static PHP_FUNCTION(session_destroy)
{
	if (zend_parse_parameters_none() == FAILURE) {
		return;
	}

	RETURN_BOOL(php_session_destroy() == SUCCESS);
}


PHPAPI zval* php_set_session_var(zend_string *name, zval *state_val, php_unserialize_data_t *var_hash) /* {{{ */
{
	IF_SESSION_VARS() {
		zval *sess_var = Z_REFVAL(PS(http_session_vars));
		SEPARATE_ARRAY(sess_var);
		return zend_hash_update(Z_ARRVAL_P(sess_var), name, state_val);
	}
	return NULL;
}


static PHP_INI_MH(OnUpdateSessionString) /* {{{ */
{
	SESSION_CHECK_ACTIVE_STATE;
	SESSION_CHECK_OUTPUT_STATE;
	return OnUpdateString(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage);
}


PHPAPI int php_session_update_timestamp(PS_UPDATE_TIMESTAMP_ARGS) {
	return SUCCESS;
}


static void php_session_rfc1867_cleanup(php_session_rfc1867_progress *progress) /* {{{ */
{
	php_session_initialize();
	PS(session_status) = php_session_active;
	IF_SESSION_VARS() {
		zval *sess_var = Z_REFVAL(PS(http_session_vars));
		SEPARATE_ARRAY(sess_var);
		zend_hash_del(Z_ARRVAL_P(sess_var), progress->key.s);
	}
	php_session_flush(1);
} /* }}} */


static int php_session_start_set_ini(zend_string *varname, zend_string *new_value) {
	int ret;
	smart_str buf ={0};
	smart_str_appends(&buf, "session");
	smart_str_appendc(&buf, '.');
	smart_str_append(&buf, varname);
	smart_str_0(&buf);
	ret = zend_alter_ini_entry_ex(buf.s, new_value, PHP_INI_USER, PHP_INI_STAGE_RUNTIME, 0);
	smart_str_free(&buf);
	return ret;
}


static PHP_RSHUTDOWN_FUNCTION(session) /* {{{ */
{
	int i;

	if (PS(session_status) == php_session_active) {
		zend_try {
			php_session_flush(1);
		} zend_end_try();
	}
	php_rshutdown_session_globals();

	/* this should NOT be done in php_rshutdown_session_globals() */
	for (i = 0; i < PS_NUM_APIS; i++) {
		if (!Z_ISUNDEF(PS(mod_user_names).names[i])) {
			zval_ptr_dtor(&PS(mod_user_names).names[i]);
			ZVAL_UNDEF(&PS(mod_user_names).names[i]);
		}
	}

	return SUCCESS;
}


PHP_METHOD(Phar, running)
{
	char *fname, *arch, *entry;
	int fname_len, arch_len, entry_len;
	zend_bool retphar = 1;

	if (zend_parse_parameters(ZEND_NUM_ARGS(), "|b", &retphar) == FAILURE) {
		return;
	}

	fname = (char*)zend_get_executed_filename();
	fname_len = (int)strlen(fname);

	if (fname_len > 7 && !memcmp(fname, "phar://", 7) && SUCCESS == phar_split_fname(fname, fname_len, &arch, &arch_len, &entry, &entry_len, 2, 0)) {
		efree(entry);
		if (retphar) {
			RETVAL_STRINGL(fname, arch_len + 7);
			efree(arch);
			return;
		} else {
			// TODO: avoid reallocation ???
			RETVAL_STRINGL(arch, arch_len);
			efree(arch);
			return;
		}
	}

	RETURN_EMPTY_STRING();
}


static int phar_set_compression(zval *zv, void *argument) /* {{{ */
{
	phar_entry_info *entry = (phar_entry_info *)Z_PTR_P(zv);
	uint32_t compress = *(uint32_t *)argument;

	if (entry->is_deleted) {
		return ZEND_HASH_APPLY_KEEP;
	}

	entry->old_flags = entry->flags;
	entry->flags &= ~PHAR_ENT_COMPRESSION_MASK;
	entry->flags |= compress;
	entry->is_modified = 1;
	return ZEND_HASH_APPLY_KEEP;
}


bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
				    struct net_device *netdev)
{
	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);

	if (tunnel && tunnel->can_offload(priv))
		return true;
	else
		return false;
}


static struct net_device *inet6_fib_lookup_dev(struct net *net,
					       const void *addr)
{
	struct net_device *dev;
	struct dst_entry *dst;
	struct flowi6 fl6;

	if (!ipv6_stub)
		return ERR_PTR(-EAFNOSUPPORT);

	memset(&fl6, 0, sizeof(fl6));
	memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
	dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
	if (IS_ERR(dst))
		return ERR_CAST(dst);

	dev = dst->dev;
	dev_hold(dev);
	dst_release(dst);

	return dev;
}


static int __vxlan_fdb_delete(struct vxlan_dev *vxlan,
			      const unsigned char *addr, union vxlan_addr ip,
			      __be16 port, __be32 src_vni, __be32 vni,
			      u32 ifindex, bool swdev_notify)
{
	struct vxlan_fdb *f;
	struct vxlan_rdst *rd = NULL;
	int err = -ENOENT;

	f = vxlan_find_mac(vxlan, addr, src_vni);
	if (!f)
		return err;

	if (!vxlan_addr_any(&ip)) {
		rd = vxlan_fdb_find_rdst(f, &ip, port, vni, ifindex);
		if (!rd)
			goto out;
	}

	/* remove a destination if it's not the only one on the list,
	 * otherwise destroy the fdb entry
	 */
	if (rd && !list_is_singular(&f->remotes)) {
		vxlan_fdb_dst_destroy(vxlan, f, rd, swdev_notify);
		goto out;
	}

	vxlan_fdb_destroy(vxlan, f, true, swdev_notify);

out:
	return 0;
}


static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
{
	struct udphdr *udph;
	struct net_device *ndev = skb->dev;
	struct net_device *rdev = ndev;
	struct rxe_dev *rxe = rxe_get_dev_from_net(ndev);
	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);

	if (!rxe && is_vlan_dev(rdev)) {
		rdev = vlan_dev_real_dev(ndev);
		rxe = rxe_get_dev_from_net(rdev);
	}
	if (!rxe)
		goto drop;

	if (skb_linearize(skb)) {
		pr_err("skb_linearize failed\n");
		ib_device_put(&rxe->ib_dev);
		goto drop;
	}

	udph = udp_hdr(skb);
	pkt->rxe = rxe;
	pkt->port_num = 1;
	pkt->hdr = (u8 *)(udph + 1);
	pkt->mask = RXE_GRH_MASK;
	pkt->paylen = be16_to_cpu(udph->len) - sizeof(*udph);

	rxe_rcv(skb);

	/*
	 * FIXME: this is in the wrong place, it needs to be done when pkt is
	 * destroyed
	 */
	ib_device_put(&rxe->ib_dev);

	return 0;
drop:
	kfree_skb(skb);

	return 0;
}


struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
					u8 name_assign_type, u16 dst_port)
{
	struct nlattr *tb[IFLA_MAX + 1];
	struct ip_tunnel_info info;
	struct net_device *dev;
	LIST_HEAD(list_kill);
	int err;

	memset(tb, 0, sizeof(tb));
	dev = rtnl_create_link(net, name, name_assign_type,
			       &geneve_link_ops, tb, NULL);
	if (IS_ERR(dev))
		return dev;

	init_tnl_info(&info, dst_port);
	err = geneve_configure(net, dev, NULL, &info,
			       true, true, false, GENEVE_DF_UNSET);
	if (err) {
		free_netdev(dev);
		return ERR_PTR(err);
	}

	/* openvswitch users expect packet sizes to be unrestricted,
	 * so set the largest MTU we can.
	 */
	err = geneve_change_mtu(dev, IP_MAX_MTU);
	if (err)
		goto err;

	err = rtnl_configure_link(dev, NULL);
	if (err < 0)
		goto err;

	return dev;
err:
	geneve_dellink(dev, &list_kill);
	unregister_netdevice_many(&list_kill);
	return ERR_PTR(err);
}


static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)
{
	struct nlattr *tb[LS_NLA_TYPE_MAX] = {};
	int ret;

	if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
		return false;

	ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
				   nlmsg_len(nlh), ib_nl_addr_policy, NULL);
	if (ret)
		return false;

	return true;
}


static int rxe_notify(struct notifier_block *not_blk,
		      unsigned long event,
		      void *arg)
{
	struct net_device *ndev = netdev_notifier_info_to_dev(arg);
	struct rxe_dev *rxe = rxe_get_dev_from_net(ndev);

	if (!rxe)
		return NOTIFY_OK;

	switch (event) {
	case NETDEV_UNREGISTER:
		ib_unregister_device_queued(&rxe->ib_dev);
		break;
	case NETDEV_UP:
		rxe_port_up(rxe);
		break;
	case NETDEV_DOWN:
		rxe_port_down(rxe);
		break;
	case NETDEV_CHANGEMTU:
		pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
		rxe_set_mtu(rxe, ndev->mtu);
		break;
	case NETDEV_CHANGE:
		rxe_set_port_state(rxe);
		break;
	case NETDEV_REBOOT:
	case NETDEV_GOING_DOWN:
	case NETDEV_CHANGEADDR:
	case NETDEV_CHANGENAME:
	case NETDEV_FEAT_CHANGE:
	default:
		pr_info("ignoring netdev event = %ld for %s\n",
			event, ndev->name);
		break;
	}

	ib_device_put(&rxe->ib_dev);
	return NOTIFY_OK;
}


static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
{
	return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
}


static int dst_fetch_ha(const struct dst_entry *dst,
			struct rdma_dev_addr *dev_addr,
			const void *daddr)
{
	struct neighbour *n;
	int ret = 0;

	n = dst_neigh_lookup(dst, daddr);
	if (!n)
		return -ENODATA;

	if (!(n->nud_state & NUD_VALID)) {
		neigh_event_send(n, NULL);
		ret = -ENODATA;
	} else {
		neigh_ha_snapshot(dev_addr->dst_dev_addr, n, dst->dev);
	}

	neigh_release(n);

	return ret;
}


static void vxlan_cleanup(struct timer_list *t)
{
	struct vxlan_dev *vxlan = from_timer(vxlan, t, age_timer);
	unsigned long next_timer = jiffies + FDB_AGE_INTERVAL;
	unsigned int h;

	if (!netif_running(vxlan->dev))
		return;

	for (h = 0; h < FDB_HASH_SIZE; ++h) {
		struct hlist_node *p, *n;

		spin_lock(&vxlan->hash_lock[h]);
		hlist_for_each_safe(p, n, &vxlan->fdb_head[h]) {
			struct vxlan_fdb *f
				= container_of(p, struct vxlan_fdb, hlist);
			unsigned long timeout;

			if (f->state & (NUD_PERMANENT | NUD_NOARP))
				continue;

			if (f->flags & NTF_EXT_LEARNED)
				continue;

			timeout = f->used + vxlan->cfg.age_interval * HZ;
			if (time_before_eq(timeout, jiffies)) {
				netdev_dbg(vxlan->dev,
					   "garbage collect %pM\n",
					   f->eth_addr);
				f->state = NUD_STALE;
				vxlan_fdb_destroy(vxlan, f, true, true);
			} else if (time_before(timeout, next_timer))
				next_timer = timeout;
		}
		spin_unlock(&vxlan->hash_lock[h]);
	}

	mod_timer(&vxlan->age_timer, next_timer);
}


static int xmit_check_hhlen(struct sk_buff *skb)
{
	int hh_len = skb_dst(skb)->dev->hard_header_len;

	if (skb_headroom(skb) < hh_len) {
		int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));

		if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC))
			return -ENOMEM;
	}

	return 0;
}


static void vxlan_fdb_delete_default(struct vxlan_dev *vxlan, __be32 vni)
{
	struct vxlan_fdb *f;
	u32 hash_index = fdb_head_index(vxlan, all_zeros_mac, vni);

	spin_lock_bh(&vxlan->hash_lock[hash_index]);
	f = __vxlan_find_mac(vxlan, all_zeros_mac, vni);
	if (f)
		vxlan_fdb_destroy(vxlan, f, true, true);
	spin_unlock_bh(&vxlan->hash_lock[hash_index]);
}


static struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
					     struct sk_buff *skb)
{
	u32 hash = 0;
	int nh_index = 0;
	int n = 0;
	u8 alive;

	/* No need to look further into packet if there's only
	 * one path
	 */
	if (rt->rt_nhn == 1)
		return rt->rt_nh;

	alive = READ_ONCE(rt->rt_nhn_alive);
	if (alive == 0)
		return NULL;

	hash = mpls_multipath_hash(rt, skb);
	nh_index = hash % alive;
	if (alive == rt->rt_nhn)
		goto out;
	for_nexthops(rt) {
		unsigned int nh_flags = READ_ONCE(nh->nh_flags);

		if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
			continue;
		if (n == nh_index)
			return nh;
		n++;
	} endfor_nexthops(rt);

out:
	return mpls_get_nexthop(rt, nh_index);
}


static int vxlan_change_mtu(struct net_device *dev, int new_mtu)
{
	struct vxlan_dev *vxlan = netdev_priv(dev);
	struct vxlan_rdst *dst = &vxlan->default_dst;
	struct net_device *lowerdev = __dev_get_by_index(vxlan->net,
							 dst->remote_ifindex);
	bool use_ipv6 = !!(vxlan->cfg.flags & VXLAN_F_IPV6);

	/* This check is different than dev->max_mtu, because it looks at
	 * the lowerdev->mtu, rather than the static dev->max_mtu
	 */
	if (lowerdev) {
		int max_mtu = lowerdev->mtu -
			      (use_ipv6 ? VXLAN6_HEADROOM : VXLAN_HEADROOM);
		if (new_mtu > max_mtu)
			return -EINVAL;
	}

	dev->mtu = new_mtu;
	return 0;
}


static void geneve_build_header(struct genevehdr *geneveh,
				const struct ip_tunnel_info *info)
{
	geneveh->ver = GENEVE_VER;
	geneveh->opt_len = info->options_len / 4;
	geneveh->oam = !!(info->key.tun_flags & TUNNEL_OAM);
	geneveh->critical = !!(info->key.tun_flags & TUNNEL_CRIT_OPT);
	geneveh->rsvd1 = 0;
	tunnel_id_to_vni(info->key.tun_id, geneveh->vni);
	geneveh->proto_type = htons(ETH_P_TEB);
	geneveh->rsvd2 = 0;

	if (info->key.tun_flags & TUNNEL_GENEVE_OPT)
		ip_tunnel_info_opts_get(geneveh->options, info);
}


static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
				      struct mlx5e_encap_entry *e)
{
	if (!e->tunnel) {
		pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
		return -EOPNOTSUPP;
	}

	return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
}


static int mpls_nh_build_multi(struct mpls_route_config *cfg,
			       struct mpls_route *rt, u8 max_labels,
			       struct netlink_ext_ack *extack)
{
	struct rtnexthop *rtnh = cfg->rc_mp;
	struct nlattr *nla_via, *nla_newdst;
	int remaining = cfg->rc_mp_len;
	int err = 0;
	u8 nhs = 0;

	change_nexthops(rt) {
		int attrlen;

		nla_via = NULL;
		nla_newdst = NULL;

		err = -EINVAL;
		if (!rtnh_ok(rtnh, remaining))
			goto errout;

		/* neither weighted multipath nor any flags
		 * are supported
		 */
		if (rtnh->rtnh_hops || rtnh->rtnh_flags)
			goto errout;

		attrlen = rtnh_attrlen(rtnh);
		if (attrlen > 0) {
			struct nlattr *attrs = rtnh_attrs(rtnh);

			nla_via = nla_find(attrs, attrlen, RTA_VIA);
			nla_newdst = nla_find(attrs, attrlen, RTA_NEWDST);
		}

		err = mpls_nh_build(cfg->rc_nlinfo.nl_net, rt, nh,
				    rtnh->rtnh_ifindex, nla_via, nla_newdst,
				    max_labels, extack);
		if (err)
			goto errout;

		if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
			rt->rt_nhn_alive--;

		rtnh = rtnh_next(rtnh, &remaining);
		nhs++;
	} endfor_nexthops(rt);

	rt->rt_nhn = nhs;

	return 0;

errout:
	return err;
}


static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
{
	struct vxlan_dev *vxlan = netdev_priv(dev);
	const struct in6_addr *daddr;
	const struct ipv6hdr *iphdr;
	struct inet6_dev *in6_dev;
	struct neighbour *n;
	struct nd_msg *msg;

	in6_dev = __in6_dev_get(dev);
	if (!in6_dev)
		goto out;

	iphdr = ipv6_hdr(skb);
	daddr = &iphdr->daddr;
	msg = (struct nd_msg *)(iphdr + 1);

	if (ipv6_addr_loopback(daddr) ||
	    ipv6_addr_is_multicast(&msg->target))
		goto out;

	n = neigh_lookup(ipv6_stub->nd_tbl, &msg->target, dev);

	if (n) {
		struct vxlan_fdb *f;
		struct sk_buff *reply;

		if (!(n->nud_state & NUD_CONNECTED)) {
			neigh_release(n);
			goto out;
		}

		f = vxlan_find_mac(vxlan, n->ha, vni);
		if (f && vxlan_addr_any(&(first_remote_rcu(f)->remote_ip))) {
			/* bridge-local neighbor */
			neigh_release(n);
			goto out;
		}

		reply = vxlan_na_create(skb, n,
					!!(f ? f->flags & NTF_ROUTER : 0));

		neigh_release(n);

		if (reply == NULL)
			goto out;

		if (netif_rx_ni(reply) == NET_RX_DROP)
			dev->stats.rx_dropped++;

	} else if (vxlan->cfg.flags & VXLAN_F_L3MISS) {
		union vxlan_addr ipa = {
			.sin6.sin6_addr = msg->target,
			.sin6.sin6_family = AF_INET6,
		};

		vxlan_ip_miss(dev, &ipa);
	}

out:
	consume_skb(skb);
	return NETDEV_TX_OK;
}


static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
{
	return (u8 *)nh + rt->rt_via_offset;
}


static int tipc_udp_xmit(struct net *net, struct sk_buff *skb,
			 struct udp_bearer *ub, struct udp_media_addr *src,
			 struct udp_media_addr *dst, struct dst_cache *cache)
{
	struct dst_entry *ndst = dst_cache_get(cache);
	int ttl, err = 0;

	if (dst->proto == htons(ETH_P_IP)) {
		struct rtable *rt = (struct rtable *)ndst;

		if (!rt) {
			struct flowi4 fl = {
				.daddr = dst->ipv4.s_addr,
				.saddr = src->ipv4.s_addr,
				.flowi4_mark = skb->mark,
				.flowi4_proto = IPPROTO_UDP
			};
			rt = ip_route_output_key(net, &fl);
			if (IS_ERR(rt)) {
				err = PTR_ERR(rt);
				goto tx_error;
			}
			dst_cache_set_ip4(cache, &rt->dst, fl.saddr);
		}

		ttl = ip4_dst_hoplimit(&rt->dst);
		udp_tunnel_xmit_skb(rt, ub->ubsock->sk, skb, src->ipv4.s_addr,
				    dst->ipv4.s_addr, 0, ttl, 0, src->port,
				    dst->port, false, true);
#if IS_ENABLED(CONFIG_IPV6)
	} else {
		if (!ndst) {
			struct flowi6 fl6 = {
				.flowi6_oif = ub->ifindex,
				.daddr = dst->ipv6,
				.saddr = src->ipv6,
				.flowi6_proto = IPPROTO_UDP
			};
			ndst = ipv6_stub->ipv6_dst_lookup_flow(net,
							       ub->ubsock->sk,
							       &fl6, NULL);
			if (IS_ERR(ndst)) {
				err = PTR_ERR(ndst);
				goto tx_error;
			}
			dst_cache_set_ip6(cache, ndst, &fl6.saddr);
		}
		ttl = ip6_dst_hoplimit(ndst);
		err = udp_tunnel6_xmit_skb(ndst, ub->ubsock->sk, skb, NULL,
					   &src->ipv6, &dst->ipv6, 0, ttl, 0,
					   src->port, dst->port, false);
#endif
	}
	return err;

tx_error:
	kfree_skb(skb);
	return err;
}


static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
			     struct nlattr *data[],
			     struct netlink_ext_ack *extack)
{
	struct geneve_dev *geneve = netdev_priv(dev);
	struct geneve_sock *gs4, *gs6;
	struct ip_tunnel_info info;
	bool metadata;
	bool use_udp6_rx_checksums;
	enum ifla_geneve_df df;
	bool ttl_inherit;
	int err;

	/* If the geneve device is configured for metadata (or externally
	 * controlled, for example, OVS), then nothing can be changed.
	 */
	if (geneve->collect_md)
		return -EOPNOTSUPP;

	/* Start with the existing info. */
	memcpy(&info, &geneve->info, sizeof(info));
	metadata = geneve->collect_md;
	use_udp6_rx_checksums = geneve->use_udp6_rx_checksums;
	ttl_inherit = geneve->ttl_inherit;
	err = geneve_nl2info(tb, data, extack, &info, &metadata,
			     &use_udp6_rx_checksums, &ttl_inherit, &df, true);
	if (err)
		return err;

	if (!geneve_dst_addr_equal(&geneve->info, &info)) {
		dst_cache_reset(&info.dst_cache);
		geneve_link_config(dev, &info, tb);
	}

	geneve_quiesce(geneve, &gs4, &gs6);
	geneve->info = info;
	geneve->collect_md = metadata;
	geneve->use_udp6_rx_checksums = use_udp6_rx_checksums;
	geneve->ttl_inherit = ttl_inherit;
	geneve_unquiesce(geneve, gs4, gs6);

	return 0;
}


static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct vxlan_dev *vxlan = netdev_priv(dev);
	struct vxlan_rdst *rdst, *fdst = NULL;
	const struct ip_tunnel_info *info;
	bool did_rsc = false;
	struct vxlan_fdb *f;
	struct ethhdr *eth;
	__be32 vni = 0;

	info = skb_tunnel_info(skb);

	skb_reset_mac_header(skb);

	if (vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) {
		if (info && info->mode & IP_TUNNEL_INFO_BRIDGE &&
		    info->mode & IP_TUNNEL_INFO_TX) {
			vni = tunnel_id_to_key32(info->key.tun_id);
		} else {
			if (info && info->mode & IP_TUNNEL_INFO_TX)
				vxlan_xmit_one(skb, dev, vni, NULL, false);
			else
				kfree_skb(skb);
			return NETDEV_TX_OK;
		}
	}

	if (vxlan->cfg.flags & VXLAN_F_PROXY) {
		eth = eth_hdr(skb);
		if (ntohs(eth->h_proto) == ETH_P_ARP)
			return arp_reduce(dev, skb, vni);
#if IS_ENABLED(CONFIG_IPV6)
		else if (ntohs(eth->h_proto) == ETH_P_IPV6 &&
			 pskb_may_pull(skb, sizeof(struct ipv6hdr) +
					    sizeof(struct nd_msg)) &&
			 ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6) {
			struct nd_msg *m = (struct nd_msg *)(ipv6_hdr(skb) + 1);

			if (m->icmph.icmp6_code == 0 &&
			    m->icmph.icmp6_type == NDISC_NEIGHBOUR_SOLICITATION)
				return neigh_reduce(dev, skb, vni);
		}
#endif
	}

	eth = eth_hdr(skb);
	f = vxlan_find_mac(vxlan, eth->h_dest, vni);
	did_rsc = false;

	if (f && (f->flags & NTF_ROUTER) && (vxlan->cfg.flags & VXLAN_F_RSC) &&
	    (ntohs(eth->h_proto) == ETH_P_IP ||
	     ntohs(eth->h_proto) == ETH_P_IPV6)) {
		did_rsc = route_shortcircuit(dev, skb);
		if (did_rsc)
			f = vxlan_find_mac(vxlan, eth->h_dest, vni);
	}

	if (f == NULL) {
		f = vxlan_find_mac(vxlan, all_zeros_mac, vni);
		if (f == NULL) {
			if ((vxlan->cfg.flags & VXLAN_F_L2MISS) &&
			    !is_multicast_ether_addr(eth->h_dest))
				vxlan_fdb_miss(vxlan, eth->h_dest);

			dev->stats.tx_dropped++;
			kfree_skb(skb);
			return NETDEV_TX_OK;
		}
	}

	list_for_each_entry_rcu(rdst, &f->remotes, list) {
		struct sk_buff *skb1;

		if (!fdst) {
			fdst = rdst;
			continue;
		}
		skb1 = skb_clone(skb, GFP_ATOMIC);
		if (skb1)
			vxlan_xmit_one(skb1, dev, vni, rdst, did_rsc);
	}

	if (fdst)
		vxlan_xmit_one(skb, dev, vni, fdst, did_rsc);
	else
		kfree_skb(skb);
	return NETDEV_TX_OK;
}


static struct net_device *inet_fib_lookup_dev(struct net *net,
					      const void *addr)
{
	return ERR_PTR(-EAFNOSUPPORT);
}


static inline struct bpf_lwt *bpf_lwt_lwtunnel(struct lwtunnel_state *lwt)
{
	return (struct bpf_lwt *)lwt->data;
}


static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index)
{
	struct mpls_route *rt = NULL;

	if (index < net->mpls.platform_labels) {
		struct mpls_route __rcu **platform_label =
			rcu_dereference(net->mpls.platform_label);
		rt = rcu_dereference(platform_label[index]);
	}
	return rt;
}


static bool has_gateway(const struct dst_entry *dst, sa_family_t family)
{
	struct rtable *rt;
	struct rt6_info *rt6;

	if (family == AF_INET) {
		rt = container_of(dst, struct rtable, dst);
		return rt->rt_uses_gateway;
	}

	rt6 = container_of(dst, struct rt6_info, dst);
	return rt6->rt6i_flags & RTF_GATEWAY;
}


static void vxlan_set_multicast_list(struct net_device *dev)
{
}


static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
			   struct netlink_ext_ack *extack)
{
	if (tb[IFLA_ADDRESS]) {
		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
					    "Provided link layer address is not Ethernet");
			return -EINVAL;
		}

		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
					    "Provided Ethernet address is not unicast");
			return -EADDRNOTAVAIL;
		}
	}

	if (!data) {
		NL_SET_ERR_MSG(extack,
			       "Not enough attributes provided to perform the operation");
		return -EINVAL;
	}

	if (data[IFLA_GENEVE_ID]) {
		__u32 vni =  nla_get_u32(data[IFLA_GENEVE_ID]);

		if (vni >= GENEVE_N_VID) {
			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_ID],
					    "Geneve ID must be lower than 16777216");
			return -ERANGE;
		}
	}

	if (data[IFLA_GENEVE_DF]) {
		enum ifla_geneve_df df = nla_get_u8(data[IFLA_GENEVE_DF]);

		if (df < 0 || df > GENEVE_DF_MAX) {
			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_GENEVE_DF],
					    "Invalid DF attribute");
			return -EINVAL;
		}
	}

	return 0;
}


static int __init inet6_init(void)
{
	struct list_head *r;
	int err = 0;

	BUILD_BUG_ON(sizeof(struct inet6_skb_parm) > FIELD_SIZEOF(struct sk_buff, cb));

	/* Register the socket-side information for inet6_create.  */
	for (r = &inetsw6[0]; r < &inetsw6[SOCK_MAX]; ++r)
		INIT_LIST_HEAD(r);

	if (disable_ipv6_mod) {
		pr_info("Loaded, but administratively disabled, reboot required to enable\n");
		goto out;
	}

	err = proto_register(&tcpv6_prot, 1);
	if (err)
		goto out;

	err = proto_register(&udpv6_prot, 1);
	if (err)
		goto out_unregister_tcp_proto;

	err = proto_register(&udplitev6_prot, 1);
	if (err)
		goto out_unregister_udp_proto;

	err = proto_register(&rawv6_prot, 1);
	if (err)
		goto out_unregister_udplite_proto;

	err = proto_register(&pingv6_prot, 1);
	if (err)
		goto out_unregister_ping_proto;

	/* We MUST register RAW sockets before we create the ICMP6,
	 * IGMP6, or NDISC control sockets.
	 */
	err = rawv6_init();
	if (err)
		goto out_unregister_raw_proto;

	/* Register the family here so that the init calls below will
	 * be able to create sockets. (?? is this dangerous ??)
	 */
	err = sock_register(&inet6_family_ops);
	if (err)
		goto out_sock_register_fail;

	tcpv6_prot.sysctl_mem = init_net.ipv4.sysctl_tcp_mem;

	/*
	 *	ipngwg API draft makes clear that the correct semantics
	 *	for TCP and UDP is to consider one TCP and UDP instance
	 *	in a host available by both INET and INET6 APIs and
	 *	able to communicate via both network protocols.
	 */

	err = register_pernet_subsys(&inet6_net_ops);
	if (err)
		goto register_pernet_fail;
	err = icmpv6_init();
	if (err)
		goto icmp_fail;
	err = ip6_mr_init();
	if (err)
		goto ipmr_fail;
	err = ndisc_init();
	if (err)
		goto ndisc_fail;
	err = igmp6_init();
	if (err)
		goto igmp_fail;

	ipv6_stub = &ipv6_stub_impl;

	err = ipv6_netfilter_init();
	if (err)
		goto netfilter_fail;
	/* Create /proc/foo6 entries. */
#ifdef CONFIG_PROC_FS
	err = -ENOMEM;
	if (raw6_proc_init())
		goto proc_raw6_fail;
	if (udplite6_proc_init())
		goto proc_udplite6_fail;
	if (ipv6_misc_proc_init())
		goto proc_misc6_fail;
	if (if6_proc_init())
		goto proc_if6_fail;
#endif
	err = ip6_route_init();
	if (err)
		goto ip6_route_fail;
	err = ip6_flowlabel_init();
	if (err)
		goto ip6_flowlabel_fail;
	err = addrconf_init();
	if (err)
		goto addrconf_fail;

	/* Init v6 extension headers. */
	err = ipv6_exthdrs_init();
	if (err)
		goto ipv6_exthdrs_fail;

	err = ipv6_frag_init();
	if (err)
		goto ipv6_frag_fail;

	/* Init v6 transport protocols. */
	err = udpv6_init();
	if (err)
		goto udpv6_fail;

	err = udplitev6_init();
	if (err)
		goto udplitev6_fail;

	err = tcpv6_init();
	if (err)
		goto tcpv6_fail;

	err = ipv6_packet_init();
	if (err)
		goto ipv6_packet_fail;

	err = pingv6_init();
	if (err)
		goto pingv6_fail;

#ifdef CONFIG_SYSCTL
	err = ipv6_sysctl_register();
	if (err)
		goto sysctl_fail;
#endif
out:
	return err;

#ifdef CONFIG_SYSCTL
sysctl_fail:
	ipv6_packet_cleanup();
#endif
pingv6_fail:
	pingv6_exit();
ipv6_packet_fail:
	tcpv6_exit();
tcpv6_fail:
	udplitev6_exit();
udplitev6_fail:
	udpv6_exit();
udpv6_fail:
	ipv6_frag_exit();
ipv6_frag_fail:
	ipv6_exthdrs_exit();
ipv6_exthdrs_fail:
	addrconf_cleanup();
addrconf_fail:
	ip6_flowlabel_cleanup();
ip6_flowlabel_fail:
	ip6_route_cleanup();
ip6_route_fail:
#ifdef CONFIG_PROC_FS
	if6_proc_exit();
proc_if6_fail:
	ipv6_misc_proc_exit();
proc_misc6_fail:
	udplite6_proc_exit();
proc_udplite6_fail:
	raw6_proc_exit();
proc_raw6_fail:
#endif
	ipv6_netfilter_fini();
netfilter_fail:
	igmp6_cleanup();
igmp_fail:
	ndisc_cleanup();
ndisc_fail:
	ip6_mr_cleanup();
ipmr_fail:
	icmpv6_cleanup();
icmp_fail:
	unregister_pernet_subsys(&inet6_net_ops);
register_pernet_fail:
	sock_unregister(PF_INET6);
	rtnl_unregister_all(PF_INET6);
out_sock_register_fail:
	rawv6_exit();
out_unregister_ping_proto:
	proto_unregister(&pingv6_prot);
out_unregister_raw_proto:
	proto_unregister(&rawv6_prot);
out_unregister_udplite_proto:
	proto_unregister(&udplitev6_prot);
out_unregister_udp_proto:
	proto_unregister(&udpv6_prot);
out_unregister_tcp_proto:
	proto_unregister(&tcpv6_prot);
	goto out;
}


uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a)
{
    return pauth_strip(env, a, false);
}


static int vnc_refresh_lossy_rect(VncDisplay *vd, int x, int y)
{
    VncState *vs;
    int sty = y / VNC_STAT_RECT;
    int stx = x / VNC_STAT_RECT;
    int has_dirty = 0;

    y = QEMU_ALIGN_DOWN(y, VNC_STAT_RECT);
    x = QEMU_ALIGN_DOWN(x, VNC_STAT_RECT);

    QTAILQ_FOREACH(vs, &vd->clients, next) {
        int j;

        /* kernel send buffers are full -> refresh later */
        if (vs->output.offset) {
            continue;
        }

        if (!vs->lossy_rect[sty][stx]) {
            continue;
        }

        vs->lossy_rect[sty][stx] = 0;
        for (j = 0; j < VNC_STAT_RECT; ++j) {
            bitmap_set(vs->dirty[y + j],
                       x / VNC_DIRTY_PIXELS_PER_BIT,
                       VNC_STAT_RECT / VNC_DIRTY_PIXELS_PER_BIT);
        }
        has_dirty++;
    }

    return has_dirty;
}


static void vnc_init_basic_info_from_remote_addr(QIOChannelSocket *ioc,
                                                 VncBasicInfo *info,
                                                 Error **errp)
{
    SocketAddress *addr = NULL;

    addr = qio_channel_socket_get_remote_address(ioc, errp);
    if (!addr) {
        return;
    }

    vnc_init_basic_info(addr, info, errp);
    qapi_free_SocketAddress(addr);
}