void assertInvalidFindProjection(const char* queryStr, const char* projStr, size_t errCode) {
    BSONObj query = fromjson(queryStr);
    BSONObj projObj = fromjson(projStr);
    QueryTestServiceContext serviceCtx;
    auto opCtx = serviceCtx.makeOperationContext();
    const boost::intrusive_ptr<ExpressionContext> expCtx(
        new ExpressionContext(opCtx.get(), std::unique_ptr<CollatorInterface>(nullptr), kTestNss));
    StatusWithMatchExpression statusWithMatcher =
        MatchExpressionParser::parse(query, std::move(expCtx));
    ASSERT_OK(statusWithMatcher.getStatus());
    std::unique_ptr<MatchExpression> queryMatchExpr = std::move(statusWithMatcher.getValue());
    ASSERT_THROWS_CODE(projection_ast::parse(expCtx,
                                             projObj,
                                             queryMatchExpr.get(),
                                             query,
                                             ProjectionPolicies::findProjectionPolicies()),
                       DBException,
                       errCode);
}


void mgetCommand(client *c) {
    int j;

    addReplyArrayLen(c,c->argc-1);
    for (j = 1; j < c->argc; j++) {
        robj *o = lookupKeyRead(c->db,c->argv[j]);
        if (o == NULL) {
            addReplyNull(c);
        } else {
            if (o->type != OBJ_STRING) {
                addReplyNull(c);
            } else {
                addReplyBulk(c,o);
            }
        }
    }
}


void setCommand(client *c) {
    int j;
    robj *expire = NULL;
    int unit = UNIT_SECONDS;
    int flags = OBJ_SET_NO_FLAGS;

    for (j = 3; j < c->argc; j++) {
        char *a = c->argv[j]->ptr;
        robj *next = (j == c->argc-1) ? NULL : c->argv[j+1];

        if ((a[0] == 'n' || a[0] == 'N') &&
            (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
            !(flags & OBJ_SET_XX))
        {
            flags |= OBJ_SET_NX;
        } else if ((a[0] == 'x' || a[0] == 'X') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_NX))
        {
            flags |= OBJ_SET_XX;
        } else if (!strcasecmp(c->argv[j]->ptr,"KEEPTTL") &&
                   !(flags & OBJ_SET_EX) && !(flags & OBJ_SET_PX))
        {
            flags |= OBJ_SET_KEEPTTL;
        } else if ((a[0] == 'e' || a[0] == 'E') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_KEEPTTL) &&
                   !(flags & OBJ_SET_PX) && next)
        {
            flags |= OBJ_SET_EX;
            unit = UNIT_SECONDS;
            expire = next;
            j++;
        } else if ((a[0] == 'p' || a[0] == 'P') &&
                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
                   !(flags & OBJ_SET_KEEPTTL) &&
                   !(flags & OBJ_SET_EX) && next)
        {
            flags |= OBJ_SET_PX;
            unit = UNIT_MILLISECONDS;
            expire = next;
            j++;
        } else {
            addReply(c,shared.syntaxerr);
            return;
        }
    }

    c->argv[2] = tryObjectEncoding(c->argv[2]);
    setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL);
}


size_t intsetBlobLen(intset *is) {
    return sizeof(intset)+(size_t)intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
}


int mutt_seqset_iterator_next (SEQSET_ITERATOR *iter, unsigned int *next)
{
  char *range_sep;

  if (!iter || !next)
    return -1;

  if (iter->in_range)
  {
    if ((iter->down && iter->range_cur == (iter->range_end - 1)) ||
        (!iter->down && iter->range_cur == (iter->range_end + 1)))
      iter->in_range = 0;
  }

  if (!iter->in_range)
  {
    iter->substr_cur = iter->substr_end;
    if (iter->substr_cur == iter->eostr)
      return 1;

    iter->substr_end = strchr (iter->substr_cur, ',');
    if (!iter->substr_end)
      iter->substr_end = iter->eostr;
    else
      *(iter->substr_end++) = '\0';

    range_sep = strchr (iter->substr_cur, ':');
    if (range_sep)
      *range_sep++ = '\0';

    if (mutt_atoui (iter->substr_cur, &iter->range_cur))
      return -1;
    if (range_sep)
    {
      if (mutt_atoui (range_sep, &iter->range_end))
        return -1;
    }
    else
      iter->range_end = iter->range_cur;

    iter->down = (iter->range_end < iter->range_cur);
    iter->in_range = 1;
  }

  *next = iter->range_cur;
  if (iter->down)
    iter->range_cur--;
  else
    iter->range_cur++;

  return 0;
}


int imap_expand_path (BUFFER* path)
{
  IMAP_MBOX mx;
  IMAP_DATA* idata;
  ciss_url_t url;
  char fixedpath[LONG_STRING];
  int rc;

  if (imap_parse_path (mutt_b2s (path), &mx) < 0)
    return -1;

  idata = imap_conn_find (&mx.account, MUTT_IMAP_CONN_NONEW);
  mutt_account_tourl (&mx.account, &url);
  imap_fix_path (idata, mx.mbox, fixedpath, sizeof (fixedpath));
  url.path = fixedpath;

  rc = url_ciss_tobuffer (&url, path, U_DECODE_PASSWD);
  FREE (&mx.mbox);

  return rc;
}


static void imap_msn_index_to_uid_seqset (BUFFER *b, IMAP_DATA *idata)
{
  int first = 1, state = 0, match = 0;
  HEADER *cur_header;
  unsigned int msn, cur_uid = 0, last_uid = 0;
  unsigned int range_begin = 0, range_end = 0;

  for (msn = 1; msn <= idata->max_msn + 1; msn++)
  {
    match = 0;
    if (msn <= idata->max_msn)
    {
      cur_header = idata->msn_index[msn - 1];
      cur_uid = cur_header ? HEADER_DATA(cur_header)->uid : 0;
      if (!state || (cur_uid && (cur_uid - 1 == last_uid)))
        match = 1;
      last_uid = cur_uid;
    }

    if (match)
    {
      switch (state)
      {
        case 1:            /* single: convert to a range */
          state = 2;
          /* fall through */
        case 2:            /* extend range ending */
          range_end = cur_uid;
          break;
        default:
          state = 1;
          range_begin = cur_uid;
          break;
      }
    }
    else if (state)
    {
      if (first)
        first = 0;
      else
        mutt_buffer_addch (b, ',');

      if (state == 1)
        mutt_buffer_add_printf (b, "%u", range_begin);
      else if (state == 2)
        mutt_buffer_add_printf (b, "%u:%u", range_begin, range_end);

      state = 1;
      range_begin = cur_uid;
    }
  }
}


void imap_make_date (char *buf, time_t timestamp)
{
  struct tm* tm = localtime (&timestamp);
  time_t tz = mutt_local_tz (timestamp);

  tz /= 60;

  snprintf (buf, IMAP_DATELEN, "%02d-%s-%d %02d:%02d:%02d %+03d%02d",
            tm->tm_mday, Months[tm->tm_mon], tm->tm_year + 1900,
            tm->tm_hour, tm->tm_min, tm->tm_sec,
            (int) tz / 60, (int) abs ((int) tz) % 60);
}


j2ku_gray_i(
    opj_image_t *in,
    const JPEG2KTILEINFO *tileinfo,
    const UINT8 *tiledata,
    Imaging im) {
    unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0;
    unsigned w = tileinfo->x1 - tileinfo->x0;
    unsigned h = tileinfo->y1 - tileinfo->y0;

    int shift = 16 - in->comps[0].prec;
    int offset = in->comps[0].sgnd ? 1 << (in->comps[0].prec - 1) : 0;
    int csiz = (in->comps[0].prec + 7) >> 3;

    unsigned x, y;

    if (csiz == 3) {
        csiz = 4;
    }

    if (shift < 0) {
        offset += 1 << (-shift - 1);
    }

    switch (csiz) {
        case 1:
            for (y = 0; y < h; ++y) {
                const UINT8 *data = &tiledata[y * w];
                UINT16 *row = (UINT16 *)im->image[y0 + y] + x0;
                for (x = 0; x < w; ++x) {
                    *row++ = j2ku_shift(offset + *data++, shift);
                }
            }
            break;
        case 2:
            for (y = 0; y < h; ++y) {
                const UINT16 *data = (const UINT16 *)&tiledata[2 * y * w];
                UINT16 *row = (UINT16 *)im->image[y0 + y] + x0;
                for (x = 0; x < w; ++x) {
                    *row++ = j2ku_shift(offset + *data++, shift);
                }
            }
            break;
        case 4:
            for (y = 0; y < h; ++y) {
                const UINT32 *data = (const UINT32 *)&tiledata[4 * y * w];
                UINT16 *row = (UINT16 *)im->image[y0 + y] + x0;
                for (x = 0; x < w; ++x) {
                    *row++ = j2ku_shift(offset + *data++, shift);
                }
            }
            break;
    }
}


vg_resource_flush(VuGpu *g,
                  struct virtio_gpu_ctrl_command *cmd)
{
    struct virtio_gpu_simple_resource *res;
    struct virtio_gpu_resource_flush rf;
    pixman_region16_t flush_region;
    int i;

    VUGPU_FILL_CMD(rf);
    virtio_gpu_bswap_32(&rf, sizeof(rf));

    res = virtio_gpu_find_resource(g, rf.resource_id);
    if (!res) {
        g_critical("%s: illegal resource specified %d\n",
                   __func__, rf.resource_id);
        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
        return;
    }

    if (rf.r.x > res->width ||
        rf.r.y > res->height ||
        rf.r.width > res->width ||
        rf.r.height > res->height ||
        rf.r.x + rf.r.width > res->width ||
        rf.r.y + rf.r.height > res->height) {
        g_critical("%s: flush bounds outside resource"
                   " bounds for resource %d: %d %d %d %d vs %d %d\n",
                   __func__, rf.resource_id, rf.r.x, rf.r.y,
                   rf.r.width, rf.r.height, res->width, res->height);
        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
        return;
    }

    pixman_region_init_rect(&flush_region,
                            rf.r.x, rf.r.y, rf.r.width, rf.r.height);
    for (i = 0; i < VIRTIO_GPU_MAX_SCANOUTS; i++) {
        struct virtio_gpu_scanout *scanout;
        pixman_region16_t region, finalregion;
        pixman_box16_t *extents;

        if (!(res->scanout_bitmask & (1 << i))) {
            continue;
        }
        scanout = &g->scanout[i];

        pixman_region_init(&finalregion);
        pixman_region_init_rect(&region, scanout->x, scanout->y,
                                scanout->width, scanout->height);

        pixman_region_intersect(&finalregion, &flush_region, &region);

        extents = pixman_region_extents(&finalregion);
        size_t width = extents->x2 - extents->x1;
        size_t height = extents->y2 - extents->y1;

        if (vugbm_buffer_can_get_dmabuf_fd(&res->buffer)) {
            VhostUserGpuMsg vmsg = {
                .request = VHOST_USER_GPU_DMABUF_UPDATE,
                .size = sizeof(VhostUserGpuUpdate),
                .payload.update = (VhostUserGpuUpdate) {
                    .scanout_id = i,
                    .x = extents->x1,
                    .y = extents->y1,
                    .width = width,
                    .height = height,
                }
            };
            vg_send_msg(g, &vmsg, -1);
            vg_wait_ok(g);
        } else {
            size_t bpp =
                PIXMAN_FORMAT_BPP(pixman_image_get_format(res->image)) / 8;
            size_t size = width * height * bpp;

            void *p = g_malloc(VHOST_USER_GPU_HDR_SIZE +
                               sizeof(VhostUserGpuUpdate) + size);
            VhostUserGpuMsg *msg = p;
            msg->request = VHOST_USER_GPU_UPDATE;
            msg->size = sizeof(VhostUserGpuUpdate) + size;
            msg->payload.update = (VhostUserGpuUpdate) {
                .scanout_id = i,
                .x = extents->x1,
                .y = extents->y1,
                .width = width,
                .height = height,
            };
            pixman_image_t *i =
                pixman_image_create_bits(pixman_image_get_format(res->image),
                                         msg->payload.update.width,
                                         msg->payload.update.height,
                                         p + offsetof(VhostUserGpuMsg,
                                                      payload.update.data),
                                         width * bpp);
            pixman_image_composite(PIXMAN_OP_SRC,
                                   res->image, NULL, i,
                                   extents->x1, extents->y1,
                                   0, 0, 0, 0,
                                   width, height);
            pixman_image_unref(i);
            vg_send_msg(g, msg, -1);
            g_free(msg);
        }
        pixman_region_fini(&region);
        pixman_region_fini(&finalregion);
    }
    pixman_region_fini(&flush_region);
}


vg_recv_msg(VuGpu *g, uint32_t expect_req, uint32_t expect_size,
            gpointer payload)
{
    uint32_t req, flags, size;

    if (vg_sock_fd_read(g->sock_fd, &req, sizeof(req)) < 0 ||
        vg_sock_fd_read(g->sock_fd, &flags, sizeof(flags)) < 0 ||
        vg_sock_fd_read(g->sock_fd, &size, sizeof(size)) < 0) {
        goto err;
    }

    g_return_val_if_fail(req == expect_req, false);
    g_return_val_if_fail(flags & VHOST_USER_GPU_MSG_FLAG_REPLY, false);
    g_return_val_if_fail(size == expect_size, false);

    if (size && vg_sock_fd_read(g->sock_fd, payload, size) != size) {
        goto err;
    }

    return true;

err:
    vg_sock_fd_close(g);
    return false;
}


vg_create_mapping_iov(VuGpu *g,
                      struct virtio_gpu_resource_attach_backing *ab,
                      struct virtio_gpu_ctrl_command *cmd,
                      struct iovec **iov)
{
    struct virtio_gpu_mem_entry *ents;
    size_t esize, s;
    int i;

    if (ab->nr_entries > 16384) {
        g_critical("%s: nr_entries is too big (%d > 16384)",
                   __func__, ab->nr_entries);
        return -1;
    }

    esize = sizeof(*ents) * ab->nr_entries;
    ents = g_malloc(esize);
    s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
                   sizeof(*ab), ents, esize);
    if (s != esize) {
        g_critical("%s: command data size incorrect %zu vs %zu",
                   __func__, s, esize);
        g_free(ents);
        return -1;
    }

    *iov = g_malloc0(sizeof(struct iovec) * ab->nr_entries);
    for (i = 0; i < ab->nr_entries; i++) {
        uint64_t len = ents[i].length;
        (*iov)[i].iov_len = ents[i].length;
        (*iov)[i].iov_base = vu_gpa_to_va(&g->dev.parent, &len, ents[i].addr);
        if (!(*iov)[i].iov_base || len != ents[i].length) {
            g_critical("%s: resource %d element %d",
                       __func__, ab->resource_id, i);
            g_free(*iov);
            g_free(ents);
            *iov = NULL;
            return -1;
        }
    }
    g_free(ents);
    return 0;
}


vg_cmd_to_string(int cmd)
{
#define CMD(cmd) [cmd] = #cmd
    static const char *vg_cmd_str[] = {
        CMD(VIRTIO_GPU_UNDEFINED),

        /* 2d commands */
        CMD(VIRTIO_GPU_CMD_GET_DISPLAY_INFO),
        CMD(VIRTIO_GPU_CMD_RESOURCE_CREATE_2D),
        CMD(VIRTIO_GPU_CMD_RESOURCE_UNREF),
        CMD(VIRTIO_GPU_CMD_SET_SCANOUT),
        CMD(VIRTIO_GPU_CMD_RESOURCE_FLUSH),
        CMD(VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D),
        CMD(VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING),
        CMD(VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING),
        CMD(VIRTIO_GPU_CMD_GET_CAPSET_INFO),
        CMD(VIRTIO_GPU_CMD_GET_CAPSET),

        /* 3d commands */
        CMD(VIRTIO_GPU_CMD_CTX_CREATE),
        CMD(VIRTIO_GPU_CMD_CTX_DESTROY),
        CMD(VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE),
        CMD(VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE),
        CMD(VIRTIO_GPU_CMD_RESOURCE_CREATE_3D),
        CMD(VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D),
        CMD(VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D),
        CMD(VIRTIO_GPU_CMD_SUBMIT_3D),

        /* cursor commands */
        CMD(VIRTIO_GPU_CMD_UPDATE_CURSOR),
        CMD(VIRTIO_GPU_CMD_MOVE_CURSOR),
    };
#undef REQ

    if (cmd >= 0 && cmd < G_N_ELEMENTS(vg_cmd_str)) {
        return vg_cmd_str[cmd];
    } else {
        return "unknown";
    }
}


vg_resource_destroy(VuGpu *g,
                    struct virtio_gpu_simple_resource *res)
{
    int i;

    if (res->scanout_bitmask) {
        for (i = 0; i < VIRTIO_GPU_MAX_SCANOUTS; i++) {
            if (res->scanout_bitmask & (1 << i)) {
                vg_disable_scanout(g, i);
            }
        }
    }

    vugbm_buffer_destroy(&res->buffer);
    g_free(res->iov);
    pixman_image_unref(res->image);
    QTAILQ_REMOVE(&g->reslist, res, next);
    g_free(res);
}


vg_virgl_get_num_capsets(void)
{
    uint32_t capset2_max_ver, capset2_max_size;
    virgl_renderer_get_cap_set(VIRTIO_GPU_CAPSET_VIRGL2,
                               &capset2_max_ver,
                               &capset2_max_size);

    return capset2_max_ver ? 2 : 1;
}


virgl_resource_detach_backing(VuGpu *g,
                              struct virtio_gpu_ctrl_command *cmd)
{
    struct virtio_gpu_resource_detach_backing detach_rb;
    struct iovec *res_iovs = NULL;
    int num_iovs = 0;

    VUGPU_FILL_CMD(detach_rb);

    virgl_renderer_resource_detach_iov(detach_rb.resource_id,
                                       &res_iovs,
                                       &num_iovs);
    if (res_iovs == NULL || num_iovs == 0) {
        return;
    }
    g_free(res_iovs);
}


virgl_cmd_get_capset(VuGpu *g,
                     struct virtio_gpu_ctrl_command *cmd)
{
    struct virtio_gpu_get_capset gc;
    struct virtio_gpu_resp_capset *resp;
    uint32_t max_ver, max_size;

    VUGPU_FILL_CMD(gc);

    virgl_renderer_get_cap_set(gc.capset_id, &max_ver,
                               &max_size);
    if (!max_size) {
        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
        return;
    }
    resp = g_malloc0(sizeof(*resp) + max_size);

    resp->hdr.type = VIRTIO_GPU_RESP_OK_CAPSET;
    virgl_renderer_fill_caps(gc.capset_id,
                             gc.capset_version,
                             (void *)resp->capset_data);
    vg_ctrl_response(g, cmd, &resp->hdr, sizeof(*resp) + max_size);
    g_free(resp);
}


static void annotation_get_freespace(annotate_state_t *state,
                                     struct annotate_entry_list *entry)
{
    uint64_t tavail = 0;
    struct buf value = BUF_INITIALIZER;

    (void) partlist_local_find_freespace_most(0, NULL, NULL, &tavail, NULL);
    buf_printf(&value, "%" PRIuMAX, (uintmax_t)tavail);
    output_entryatt(state, entry->name, "", &value);
    buf_free(&value);
}


EXPORTED void freestrlist(struct strlist *l)
{
    struct strlist *n;

    while (l) {
        n = l->next;
        free(l->s);
        if (l->p) charset_freepat(l->p);
        free((char *)l);
        l = n;
    }
}


static void annotation_get_uniqueid(annotate_state_t *state,
                                    struct annotate_entry_list *entry)
{
    struct buf value = BUF_INITIALIZER;

    assert(state->mailbox);

    if (state->mailbox->uniqueid)
        buf_appendcstr(&value, state->mailbox->uniqueid);

    output_entryatt(state, entry->name, "", &value);
    buf_free(&value);
}


EXPORTED int specialuse_validate(const char *mboxname, const char *userid,
                                 const char *src, struct buf *dest)
{
    const char *specialuse_extra_opt = config_getstring(IMAPOPT_SPECIALUSE_EXTRA);
    char *strval = NULL;
    strarray_t *valid = NULL;
    strarray_t *new_attribs = NULL;
    strarray_t *cur_attribs = NULL;
    struct buf mbattribs = BUF_INITIALIZER;
    int i, j;
    int r = 0;

    if (!src) {
        buf_reset(dest);
        return 0;
    }

    /* If there is a valid mboxname, we get the current specialuse annotations.
     */
    if (mboxname) {
        annotatemore_lookup(mboxname, "/specialuse", userid, &mbattribs);
        if (mbattribs.len) {
            cur_attribs = strarray_split(buf_cstring(&mbattribs), NULL, 0);
        }
    }

    /* check specialuse_extra option if set */
    if (specialuse_extra_opt)
        valid = strarray_split(specialuse_extra_opt, NULL, 0);
    else
        valid = strarray_new();

    /* strarray_add(valid, "\\All"); -- we don't support virtual folders right now */
    strarray_add(valid, "\\Archive");
    strarray_add(valid, "\\Drafts");
    /* strarray_add(valid, "\\Flagged"); -- we don't support virtual folders right now */
    strarray_add(valid, "\\Important"); // draft-ietf-specialuse-important
    strarray_add(valid, "\\Junk");
    strarray_add(valid, "\\Sent");
    strarray_add(valid, "\\Trash");
    strarray_add(valid, "\\Snoozed"); // JMAP

    new_attribs = strarray_split(src, NULL, 0);

    for (i = 0; i < new_attribs->count; i++) {
        int skip_mbcheck = 0;
        const char *item = strarray_nth(new_attribs, i);

        for (j = 0; j < valid->count; j++) { /* can't use find here */
            if (!strcasecmp(strarray_nth(valid, j), item))
                break;
            /* or without the leading '\' */
            if (!strcasecmp(strarray_nth(valid, j) + 1, item))
                break;
        }

        if (j >= valid->count) {
            r = IMAP_ANNOTATION_BADENTRY;
            goto done;
        }

        if (cur_attribs &&
            (strarray_find_case(cur_attribs, strarray_nth(valid, j), 0) >= 0)) {
            /* The mailbox has this specialuse attribute set already */
            skip_mbcheck = 1;
        }

        /* don't allow names that are already in use */
        if (!skip_mbcheck) {
            char *mbname = mboxlist_find_specialuse(strarray_nth(valid, j),
                                                    userid);
            if (mbname) {
                free(mbname);
                r = IMAP_MAILBOX_SPECIALUSE;
                goto done;
            }
        }

        /* normalise the value */
        strarray_set(new_attribs, i, strarray_nth(valid, j));
    }

    strval = strarray_join(new_attribs, " ");
    buf_setcstr(dest, strval);

done:
    free(strval);
    strarray_free(valid);
    strarray_free(new_attribs);
    strarray_free(cur_attribs);
    buf_free(&mbattribs);
    return r;
}


EXPORTED int annotate_state_store(annotate_state_t *state, struct entryattlist *l)
{
    int r = 0;
    struct entryattlist *e = l;
    struct attvaluelist *av;

    annotate_state_start(state);

    /* Build a list of callbacks for storing the annotations */
    while (e) {
        int attribs;
        const annotate_entrydesc_t *desc = NULL;
        struct annotate_entry_list *nentry = NULL;

        /* See if we support this entry */
        r = find_desc_store(state, e->entry, &desc);
        if (r)
            goto cleanup;

        /* Add this entry to our list only if it
           applies to our particular server type */
        if ((desc->proxytype != PROXY_ONLY)
            || proxy_store_func)
            nentry = _annotate_state_add_entry(state, desc, e->entry);

        /* See if we are allowed to set the given attributes. */
        attribs = desc->attribs;
        av = e->attvalues;
        while (av) {
            if (!strcmp(av->attrib, "value.shared")) {
                if (!(attribs & ATTRIB_VALUE_SHARED)) {
                    r = IMAP_PERMISSION_DENIED;
                    goto cleanup;
                }
                r = annotate_canon_value(&av->value,
                                         desc->type);
                if (r)
                    goto cleanup;
                if (nentry) {
                    buf_init_ro(&nentry->shared, av->value.s, av->value.len);
                    nentry->have_shared = 1;
                }
            }
            else if (!strcmp(av->attrib, "content-type.shared") ||
                     !strcmp(av->attrib, "content-type.priv")) {
                syslog(LOG_WARNING, "annotatemore_store: client used "
                                    "deprecated attribute \"%s\", ignoring",
                                    av->attrib);
            }
            else if (!strcmp(av->attrib, "value.priv")) {
                if (!(attribs & ATTRIB_VALUE_PRIV)) {
                    r = IMAP_PERMISSION_DENIED;
                    goto cleanup;
                }
                r = annotate_canon_value(&av->value,
                                         desc->type);
                if (r)
                    goto cleanup;
                if (nentry) {
                    buf_init_ro(&nentry->priv, av->value.s, av->value.len);
                    nentry->have_priv = 1;
                }
            }
            else {
                r = IMAP_PERMISSION_DENIED;
                goto cleanup;
            }

            av = av->next;
        }

        e = e->next;
    }

    if (state->which == ANNOTATION_SCOPE_SERVER) {
        r = _annotate_store_entries(state);
    }

    else if (state->which == ANNOTATION_SCOPE_MAILBOX) {
        if (proxy_store_func) {
            r = annotate_state_need_mbentry(state);
            if (r)
                goto cleanup;
            assert(state->mbentry);
        }
        else assert(state->mailbox);

        r = _annotate_store_entries(state);
        if (r)
            goto cleanup;

        state->count++;

        if (proxy_store_func && state->mbentry->server &&
            !hash_lookup(state->mbentry->server, &state->server_table)) {
            hash_insert(state->mbentry->server, (void *)0xDEADBEEF, &state->server_table);
        }

        if (!r && !state->count) r = IMAP_MAILBOX_NONEXISTENT;

        if (proxy_store_func) {
            if (!r) {
                /* proxy command to backends */
                struct proxy_rock prock = { NULL, NULL };
                prock.mbox_pat = state->mbentry->ext_name;
                prock.entryatts = l;
                hash_enumerate(&state->server_table, store_proxy, &prock);
            }
        }
    }
    else if (state->which == ANNOTATION_SCOPE_MESSAGE) {
        r = _annotate_store_entries(state);
        if (r)
            goto cleanup;
    }

cleanup:
    annotate_state_finish(state);
    return r;
}


EXPORTED int annotatemore_lookup(const char *mboxname, const char *entry,
                                 const char *userid, struct buf *value)
{
    return annotatemore_msg_lookup(mboxname, /*uid*/0, entry, userid, value);
}


static int parse_table_lookup_bitmask(const struct annotate_attrib *table,
                                      struct parse_state *state)
{
    char *token = get_token(state, ".-_/ ");
    char *p;
    int i;
    int result = 0;
    tok_t tok;

    if (!token)
        return -1;
    tok_initm(&tok, token, NULL, 0);

    while ((p = tok_next(&tok))) {
        state->context = p;
        i = table_lookup(table, p);
        if (i < 0)
            return i;
        result |= i;
    }

    return result;
}


EXPORTED int annotate_state_set_server(annotate_state_t *state)
{
    return annotate_state_set_scope(state, NULL, NULL, 0);
}


static void annotation_get_partition(annotate_state_t *state,
                                     struct annotate_entry_list *entry)
{
    struct buf value = BUF_INITIALIZER;
    int r;

    assert(state);
    assert(state->which == ANNOTATION_SCOPE_MAILBOX);
    r = annotate_state_need_mbentry(state);
    assert(r == 0);

    /* Make sure its a local mailbox */
    if (state->mbentry->server) goto out;

    /* Check ACL */
    if (!state->isadmin &&
        (!state->mbentry->acl ||
         !(cyrus_acl_myrights(state->auth_state, state->mbentry->acl) & ACL_LOOKUP)))
        goto out;

    buf_appendcstr(&value, state->mbentry->partition);

    output_entryatt(state, entry->name, "", &value);
out:
    buf_free(&value);
}


EXPORTED void annotatemore_close(void)
{
    /* close all the open databases */
    while (all_dbs_head)
        annotate_closedb(all_dbs_head);

    annotatemore_dbopen = 0;
}


EXPORTED void annotate_state_begin(annotate_state_t *state)
{
    init_internal();

    annotate_begin(state->d);
}


static void annotation_get_usercounters(annotate_state_t *state,
                                        struct annotate_entry_list *entry)
{
    struct buf value = BUF_INITIALIZER;
    struct mboxname_counters counters;
    char *mboxname = NULL;

    assert(state);
    assert(state->userid);

    mboxname = mboxname_user_mbox(state->userid, NULL);
    int r = mboxname_read_counters(mboxname, &counters);

    if (!r) buf_printf(&value, "%u %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %u",
                       counters.version, counters.highestmodseq,
                       counters.mailmodseq, counters.caldavmodseq,
                       counters.carddavmodseq, counters.notesmodseq,
                       counters.mailfoldersmodseq, counters.caldavfoldersmodseq,
                       counters.carddavfoldersmodseq, counters.notesfoldersmodseq,
                       counters.quotamodseq, counters.raclmodseq,
                       counters.uidvalidity);

    output_entryatt(state, entry->name, state->userid, &value);
    free(mboxname);
    buf_free(&value);
}


EXPORTED int annotate_state_fetch(annotate_state_t *state,
                         const strarray_t *entries, const strarray_t *attribs,
                         annotate_fetch_cb_t callback, void *rock)
{
    int i;
    struct glob *g;
    const ptrarray_t *non_db_entries;
    const annotate_entrydesc_t *db_entry;
    int r = 0;

    init_internal();

    annotate_state_start(state);
    state->callback = callback;
    state->callback_rock = rock;

    /* Build list of attributes to fetch */
    for (i = 0 ; i < attribs->count ; i++)
    {
        const char *s = attribs->data[i];
        int attribcount;

        /*
         * TODO: this is bogus.  The * and % wildcard characters applied
         * to attributes in the early drafts of the ANNOTATEMORE
         * extension, but not in later drafts where those characters are
         * actually illegal in attribute names.
         */
        g = glob_init(s, '.');

        for (attribcount = 0;
             annotation_attributes[attribcount].name;
             attribcount++) {
            if (GLOB_MATCH(g, annotation_attributes[attribcount].name)) {
                if (annotation_attributes[attribcount].entry & ATTRIB_DEPRECATED) {
                    if (strcmp(s, "*"))
                        syslog(LOG_WARNING, "annotatemore_fetch: client used "
                                            "deprecated attribute \"%s\", ignoring",
                                            annotation_attributes[attribcount].name);
                }
                else
                    state->attribs |= annotation_attributes[attribcount].entry;
            }
        }

        glob_free(&g);
    }

    if (!state->attribs)
        goto out;

    if (state->which == ANNOTATION_SCOPE_SERVER) {
        non_db_entries = &server_entries;
        db_entry = &server_db_entry;
    }
    else if (state->which == ANNOTATION_SCOPE_MAILBOX) {
        non_db_entries = &mailbox_entries;
        db_entry = &mailbox_db_entry;
    }
    else if (state->which == ANNOTATION_SCOPE_MESSAGE) {
        non_db_entries = &message_entries;
        db_entry = &message_db_entry;
    }
    else {
        syslog(LOG_ERR, "IOERROR: unknown annotation scope %d", state->which);
        r = IMAP_INTERNAL;
        goto out;
    }

    /* Build a list of callbacks for fetching the annotations */
    for (i = 0 ; i < entries->count ; i++)
    {
        const char *s = entries->data[i];
        int j;
        int check_db = 0; /* should we check the db for this entry? */

        g = glob_init(s, '/');

        for (j = 0 ; j < non_db_entries->count ; j++) {
            const annotate_entrydesc_t *desc = non_db_entries->data[j];

            if (!desc->get)
                continue;

            if (GLOB_MATCH(g, desc->name)) {
                /* Add this entry to our list only if it
                   applies to our particular server type */
                if ((desc->proxytype != PROXY_ONLY)
                    || proxy_fetch_func)
                    _annotate_state_add_entry(state, desc, desc->name);
            }

            if (!strcmp(s, desc->name)) {
                /* exact match */
                if (desc->proxytype != PROXY_ONLY) {
                    state->orig_entry = entries;  /* proxy it */
                }
                break;
            }
        }

        if (j == non_db_entries->count) {
            /* no [exact] match */
            state->orig_entry = entries;  /* proxy it */
            check_db = 1;
        }

        /* Add the db entry to our list if only if it
           applies to our particular server type */
        if (check_db &&
            ((db_entry->proxytype != PROXY_ONLY) || proxy_fetch_func)) {
            /* Add the db entry to our list */
            _annotate_state_add_entry(state, db_entry, s);
        }

        glob_free(&g);
    }

    if (state->which == ANNOTATION_SCOPE_SERVER) {
        _annotate_fetch_entries(state, /*proxy_check*/1);
    }
    else if (state->which == ANNOTATION_SCOPE_MAILBOX) {

        if (state->entry_list || proxy_fetch_func) {
            if (proxy_fetch_func) {
                r = annotate_state_need_mbentry(state);
                if (r)
                    goto out;
                assert(state->mbentry);
            }

            if (proxy_fetch_func && state->orig_entry) {
                state->orig_mailbox = state->mbentry->name;
                state->orig_attribute = attribs;
            }

            _annotate_fetch_entries(state, /*proxy_check*/1);

            if (proxy_fetch_func && state->orig_entry && state->mbentry->server &&
                !hash_lookup(state->mbentry->server, &state->server_table)) {
                /* xxx ignoring result */
                proxy_fetch_func(state->mbentry->server, state->mbentry->ext_name,
                                 state->orig_entry, state->orig_attribute);
                hash_insert(state->mbentry->server, (void *)0xDEADBEEF, &state->server_table);
            }
        }
    }
    else if (state->which == ANNOTATION_SCOPE_MESSAGE) {
        _annotate_fetch_entries(state, /*proxy_check*/0);
    }

    /* Flush last cached entry in output_entryatt() */
    flush_entryatt(state);

out:
    annotate_state_finish(state);
    return r;
}


static void annotation_get_lastpop(annotate_state_t *state,
                                   struct annotate_entry_list *entry)
{
    struct mailbox *mailbox = state->mailbox;
    char valuebuf[RFC3501_DATETIME_MAX+1];
    struct buf value = BUF_INITIALIZER;

    assert(mailbox);

    if (mailbox->i.pop3_last_login) {
        time_to_rfc3501(mailbox->i.pop3_last_login, valuebuf,
                        sizeof(valuebuf));
        buf_appendcstr(&value, valuebuf);
    }

    output_entryatt(state, entry->name, "", &value);
    buf_free(&value);
}


static void annotation_get_pop3showafter(annotate_state_t *state,
                                         struct annotate_entry_list *entry)
{
    struct mailbox *mailbox = state->mailbox;
    char valuebuf[RFC3501_DATETIME_MAX+1];
    struct buf value = BUF_INITIALIZER;

    assert(mailbox);

    if (mailbox->i.pop3_show_after)
    {
        time_to_rfc3501(mailbox->i.pop3_show_after, valuebuf, sizeof(valuebuf));
        buf_appendcstr(&value, valuebuf);
    }

    output_entryatt(state, entry->name, "", &value);
    buf_free(&value);
}


static int cleanup_cb(void *rock,
                      const char *key, size_t keylen,
                      const char *data __attribute__((unused)),
                      size_t datalen __attribute__((unused)))
{
    annotate_db_t *d = (annotate_db_t *)rock;

    return cyrusdb_delete(d->db, key, keylen, tid(d), /*force*/1);
}


EXPORTED int annotate_state_writemdata(annotate_state_t *state,
                                       const char *entry,
                                       const char *userid,
                                       const struct buf *value,
                                       const struct annotate_metadata *mdata)
{
    return write_entry(state->mailbox, state->uid, entry, userid, value,
                       /*ignorequota*/1, 0, mdata, /*maywrite*/1);
}


static int annotate_state_set_scope(annotate_state_t *state,
                                    const mbentry_t *mbentry,
                                    struct mailbox *mailbox,
                                    unsigned int uid)
{
    int r = 0;
    annotate_db_t *oldd = NULL;
    int oldwhich = state->which;

    init_internal();

    /* Carefully preserve the reference on the old DB just in case it
     * turns out to be the same as the new DB, so we avoid the overhead
     * of an unnecessary cyrusdb_open/close pair. */
    oldd = state->d;
    state->d = NULL;

    annotate_state_unset_scope(state);

    if (mbentry) {
        assert(!mailbox);
        assert(!uid);
        if (!mbentry->server) {
            /* local mailbox */
            r = mailbox_open_iwl(mbentry->name, &mailbox);
            if (r)
                goto out;
            state->ourmailbox = mailbox;
        }
        state->mbentry = mbentry;
        state->which = ANNOTATION_SCOPE_MAILBOX;
    }

    else if (uid) {
        assert(mailbox);
        state->which = ANNOTATION_SCOPE_MESSAGE;
    }
    else if (mailbox) {
        assert(!uid);
        state->which = ANNOTATION_SCOPE_MAILBOX;
    }
    else {
        assert(!mailbox);
        assert(!uid);
        state->which = ANNOTATION_SCOPE_SERVER;
    }
    assert(oldwhich == ANNOTATION_SCOPE_UNKNOWN ||
           oldwhich == state->which);
    state->mailbox = mailbox;
    state->uid = uid;

    r = _annotate_getdb(mailbox ? mailbox->name : NULL, uid,
                        CYRUSDB_CREATE, &state->d);

out:
    annotate_putdb(&oldd);
    return r;
}


static int normalise_attribs(struct parse_state *state, int attribs)
{
    int nattribs = 0;
    static int deprecated_warnings = 0;

    /* always provide size.shared if value.shared specified */
    if ((attribs & ATTRIB_VALUE_SHARED))
        nattribs |= ATTRIB_VALUE_SHARED|ATTRIB_SIZE_SHARED;

    /* likewise size.priv */
    if ((attribs & ATTRIB_VALUE_PRIV))
        nattribs |= ATTRIB_VALUE_PRIV|ATTRIB_SIZE_PRIV;

    /* ignore any other specified attributes */

    if ((attribs & ATTRIB_DEPRECATED)) {
        if (!deprecated_warnings++)
            parse_error(state, "deprecated attribute names such as "
                                "content-type or modified-since (ignoring)");
    }

    return nattribs;
}


static const char *key_as_string(const annotate_db_t *d,
                                 const char *key, int keylen)
{
    const char *mboxname, *entry, *userid;
    unsigned int uid;
    int r;
    static struct buf buf = BUF_INITIALIZER;

    buf_reset(&buf);
    r = split_key(d, key, keylen, &mboxname, &uid, &entry, &userid);
    if (r)
        buf_appendcstr(&buf, "invalid");
    else
        buf_printf(&buf, "{ mboxname=\"%s\" uid=%u entry=\"%s\" userid=\"%s\" }",
                   mboxname, uid, entry, userid);
    return buf_cstring(&buf);
}


static void annotation_get_synccrcs(annotate_state_t *state,
                                    struct annotate_entry_list *entry)
{
    struct mailbox *mailbox = state->mailbox;
    struct buf value = BUF_INITIALIZER;

    assert(mailbox);

    buf_printf(&value, "%u %u", mailbox->i.synccrcs.basic,
                                mailbox->i.synccrcs.annot);

    output_entryatt(state, entry->name, "", &value);
    buf_free(&value);
}


httpHdrContRangePackInto(const HttpHdrContRange * range, Packable * p)
{
    assert(range && p);
    httpHdrRangeRespSpecPackInto(&range->spec, p);
    /* Ensure typecast is safe */
    assert (range->elength >= 0);

    if (!known_spec(range->elength))
        p->append("/*", 2);
    else
        p->appendf("/%" PRId64, range->elength);
}


purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
{
    const auto hdrUrl = rep->header.getStr(hdr);
    if (!hdrUrl)
        return;

    /*
     * If the URL is relative, make it absolute so we can find it.
     * If it's absolute, make sure the host parts match to avoid DOS attacks
     * as per RFC 2616 13.10.
     */
    SBuf absUrlMaker;
    const char *absUrl = nullptr;
    if (urlIsRelative(hdrUrl)) {
        if (req->method.id() == Http::METHOD_CONNECT)
            absUrl = hdrUrl; // TODO: merge authority-uri and hdrUrl
        else if (req->url.getScheme() == AnyP::PROTO_URN)
            absUrl = req->url.absolute().c_str();
        else {
            AnyP::Uri tmpUrl = req->url;
            if (*hdrUrl == '/') {
                // RFC 3986 section 4.2: absolute-path reference
                // for this logic replace the entire request-target URI path
                tmpUrl.path(hdrUrl);
            } else {
                tmpUrl.addRelativePath(reqUrl);
            }
            absUrlMaker = tmpUrl.absolute();
            absUrl = absUrlMaker.c_str();
        }
    } else if (!sameUrlHosts(reqUrl, hdrUrl)) {
        return;
    } else
        absUrl = hdrUrl;

    purgeEntriesByUrl(req, absUrl);
}


Http::Stream::packRange(StoreIOBuffer const &source, MemBuf *mb)
{
    HttpHdrRangeIter * i = &http->range_iter;
    Range<int64_t> available(source.range());
    char const *buf = source.data;

    while (i->currentSpec() && available.size()) {
        const size_t copy_sz = lengthToSend(available);
        if (copy_sz) {
            // intersection of "have" and "need" ranges must not be empty
            assert(http->out.offset < i->currentSpec()->offset + i->currentSpec()->length);
            assert(http->out.offset + (int64_t)available.size() > i->currentSpec()->offset);

            /*
             * put boundary and headers at the beginning of a range in a
             * multi-range
             */
            if (http->multipartRangeRequest() && i->debt() == i->currentSpec()->length) {
                clientPackRangeHdr(
                    &http->storeEntry()->mem().freshestReply(),
                    i->currentSpec(),       /* current range */
                    i->boundary,    /* boundary, the same for all */
                    mb);
            }

            // append content
            debugs(33, 3, "appending " << copy_sz << " bytes");
            noteSentBodyBytes(copy_sz);
            mb->append(buf, copy_sz);

            // update offsets
            available.start += copy_sz;
            buf += copy_sz;
        }

        if (!canPackMoreRanges()) {
            debugs(33, 3, "Returning because !canPackMoreRanges.");
            if (i->debt() == 0)
                // put terminating boundary for multiparts
                clientPackTermBound(i->boundary, mb);
            return;
        }

        int64_t nextOffset = getNextRangeOffset();
        assert(nextOffset >= http->out.offset);
        int64_t skip = nextOffset - http->out.offset;
        /* adjust for not to be transmitted bytes */
        http->out.offset = nextOffset;

        if (available.size() <= (uint64_t)skip)
            return;

        available.start += skip;
        buf += skip;

        if (copy_sz == 0)
            return;
    }
}


httpHdrContRangeSet(HttpHdrContRange * cr, HttpHdrRangeSpec spec, int64_t ent_len)
{
    assert(cr && ent_len >= 0);
    cr->spec = spec;
    cr->elength = ent_len;
}


Http::Stream::Stream(const Comm::ConnectionPointer &aConn, ClientHttpRequest *aReq) :
    clientConnection(aConn),
    http(aReq),
    reply(nullptr),
    writtenToSocket(0),
    mayUseConnection_(false),
    connRegistered_(false)
{
    assert(http != nullptr);
    memset(reqbuf, '\0', sizeof (reqbuf));
    flags.deferred = 0;
    flags.parsed_ok = 0;
    deferredparams.node = nullptr;
    deferredparams.rep = nullptr;
}


Client::addVirginReplyBody(const char *data, ssize_t len)
{
    adjustBodyBytesRead(len);

#if USE_ADAPTATION
    assert(!adaptationAccessCheckPending); // or would need to buffer while waiting
    if (startedAdaptation) {
        adaptVirginReplyBody(data, len);
        return;
    }
#endif
    storeReplyBody(data, len);
}


Client::swanSong()
{
    // get rid of our piping obligations
    if (requestBodySource != NULL)
        stopConsumingFrom(requestBodySource);

#if USE_ADAPTATION
    cleanAdaptation();
#endif

    if (!doneWithServer())
        closeServer();

    if (!doneWithFwd) {
        doneWithFwd = "swanSong()";
        fwd->handleUnregisteredServerEnd();
    }

    BodyConsumer::swanSong();
#if USE_ADAPTATION
    Initiator::swanSong();
    BodyProducer::swanSong();
#endif

    // paranoid: check that swanSong has been called
    // extra paranoid: yeah, I really mean it. they MUST pass here.
    assert(!requestBodySource);
#if USE_ADAPTATION
    assert(!virginBodyDestination);
    assert(!adaptedBodySource);
#endif
}


Client::handleAdaptedBodyProductionEnded()
{
    if (abortOnBadEntry("entry went bad while waiting for adapted body eof"))
        return;

    // end consumption if we consumed everything
    if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
        endAdaptedBodyConsumption();
    // else resumeBodyStorage() will eventually consume the rest
}


Client::handleAdaptationAborted(bool bypassable)
{
    debugs(11,5, HERE << "handleAdaptationAborted; bypassable: " << bypassable <<
           ", entry empty: " << entry->isEmpty());

    if (abortOnBadEntry("entry went bad while ICAP aborted"))
        return;

    // TODO: bypass if possible
    if (!handledEarlyAdaptationAbort())
        abortAll("adaptation failure with a filled entry");
}


Client::haveParsedReplyHeaders()
{
    Must(theFinalReply);
    maybePurgeOthers();

    // adaptation may overwrite old offset computed using the virgin response
    currentOffset = 0;
    if (const auto cr = theFinalReply->contentRange()) {
        if (cr->spec.offset != HttpHdrRangeSpec::UnknownPosition)
            currentOffset = cr->spec.offset;
    }
}


Http::Stream::packChunk(const StoreIOBuffer &bodyData, MemBuf &mb)
{
    const uint64_t length =
        static_cast<uint64_t>(lengthToSend(bodyData.range()));
    noteSentBodyBytes(length);

    mb.appendf("%" PRIX64 "\r\n", length);
    mb.append(bodyData.data, length);
    mb.append("\r\n", 2);
}


Client::abortOnData(const char *reason)
{
    abortAll(reason);
    return true;
}


httpHdrRangeRespSpecPackInto(const HttpHdrRangeSpec * spec, Packable * p)
{
    /* Ensure typecast is safe */
    assert (spec->length >= 0);

    if (!known_spec(spec->offset) || !known_spec(spec->length))
        p->append("*", 1);
    else
        p->appendf("bytes %" PRId64 "-%" PRId64, spec->offset, spec->offset + spec->length - 1);
}


Client::handleMoreAdaptedBodyAvailable()
{
    if (abortOnBadEntry("entry refuses adapted body"))
        return;

    assert(entry);

    size_t contentSize = adaptedBodySource->buf().contentSize();

    if (!contentSize)
        return; // XXX: bytesWanted asserts on zero-size ranges

    const size_t spaceAvailable = entry->bytesWanted(Range<size_t>(0, contentSize), true);

    if (spaceAvailable < contentSize ) {
        // No or partial body data consuming
        typedef NullaryMemFunT<Client> Dialer;
        AsyncCall::Pointer call = asyncCall(93, 5, "Client::resumeBodyStorage",
                                            Dialer(this, &Client::resumeBodyStorage));
        entry->deferProducer(call);
    }

    if (!spaceAvailable)  {
        debugs(11, 5, HERE << "NOT storing " << contentSize << " bytes of adapted " <<
               "response body at offset " << adaptedBodySource->consumedSize());
        return;
    }

    if (spaceAvailable < contentSize ) {
        debugs(11, 5, HERE << "postponing storage of " <<
               (contentSize - spaceAvailable) << " body bytes");
        contentSize = spaceAvailable;
    }

    debugs(11,5, HERE << "storing " << contentSize << " bytes of adapted " <<
           "response body at offset " << adaptedBodySource->consumedSize());

    BodyPipeCheckout bpc(*adaptedBodySource);
    const StoreIOBuffer ioBuf(&bpc.buf, currentOffset, contentSize);
    currentOffset += ioBuf.length;
    entry->write(ioBuf);
    bpc.buf.consume(contentSize);
    bpc.checkIn();
}


Client::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer group)
{
    adaptationAccessCheckPending = false;

    if (abortOnBadEntry("entry went bad while waiting for ICAP ACL check"))
        return;

    // TODO: Should non-ICAP and ICAP REPMOD pre-cache paths check this?
    // That check now only happens on REQMOD pre-cache and REPMOD post-cache, in processReplyAccess().
    if (virginReply()->expectedBodyTooLarge(*request)) {
        sendBodyIsTooLargeError();
        return;
    }
    // TODO: Should we check receivedBodyTooLarge as well?

    if (!group) {
        debugs(11,3, HERE << "no adapation needed");
        setFinalReply(virginReply());
        processReplyBody();
        return;
    }

    startAdaptation(group, originalRequest().getRaw());
    processReplyBody();
}


Http::Stream::initiateClose(const char *reason)
{
    debugs(33, 4, clientConnection << " because " << reason);
    getConn()->stopSending(reason); // closes ASAP
}


Client::handleMoreRequestBodyAvailable()
{
    if (!requestSender)
        sendMoreRequestBody();
    else
        debugs(9,3, HERE << "waiting for request body write to complete");
}


Client::resumeBodyStorage()
{
    if (abortOnBadEntry("store entry aborted while kick producer callback"))
        return;

    if (!adaptedBodySource)
        return;

    handleMoreAdaptedBodyAvailable();

    if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
        endAdaptedBodyConsumption();
}


Client::serverComplete2()
{
    debugs(11,5,HERE << "serverComplete2 " << this);

#if USE_ADAPTATION
    if (virginBodyDestination != NULL)
        stopProducingFor(virginBodyDestination, true);

    if (!doneWithAdaptation())
        return;
#endif

    completeForwarding();
}


Http::Stream::registerWithConn()
{
    assert(!connRegistered_);
    assert(getConn());
    connRegistered_ = true;
    getConn()->pipeline.add(Http::StreamPointer(this));
}


Http::Stream::sendStartOfMessage(HttpReply *rep, StoreIOBuffer bodyData)
{
    prepareReply(rep);
    assert(rep);
    MemBuf *mb = rep->pack();

    // dump now, so we do not output any body.
    debugs(11, 2, "HTTP Client " << clientConnection);
    debugs(11, 2, "HTTP Client REPLY:\n---------\n" << mb->buf << "\n----------");

    /* Save length of headers for persistent conn checks */
    http->out.headers_sz = mb->contentSize();
#if HEADERS_LOG
    headersLog(0, 0, http->request->method, rep);
#endif

    if (bodyData.data && bodyData.length) {
        if (multipartRangeRequest())
            packRange(bodyData, mb);
        else if (http->request->flags.chunkedReply) {
            packChunk(bodyData, *mb);
        } else {
            size_t length = lengthToSend(bodyData.range());
            noteSentBodyBytes(length);
            mb->append(bodyData.data, length);
        }
    }

    getConn()->write(mb);
    delete mb;
}


Client::~Client()
{
    // paranoid: check that swanSong has been called
    assert(!requestBodySource);
#if USE_ADAPTATION
    assert(!virginBodyDestination);
    assert(!adaptedBodySource);
#endif

    entry->unlock("Client");

    HTTPMSGUNLOCK(request);
    HTTPMSGUNLOCK(theVirginReply);
    HTTPMSGUNLOCK(theFinalReply);

    if (responseBodyBuffer != NULL) {
        delete responseBodyBuffer;
        responseBodyBuffer = NULL;
    }
}


BPF_CALL_4(bpf_ringbuf_output, struct bpf_map *, map, void *, data, u64, size,
	   u64, flags)
{
	struct bpf_ringbuf_map *rb_map;
	void *rec;

	if (unlikely(flags & ~(BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP)))
		return -EINVAL;

	rb_map = container_of(map, struct bpf_ringbuf_map, map);
	rec = __bpf_ringbuf_reserve(rb_map->rb, size);
	if (!rec)
		return -EAGAIN;

	memcpy(rec, data, size);
	bpf_ringbuf_commit(rec, flags, false /* discard */);
	return 0;
}


static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
{
	struct bpf_ringbuf_map *rb_map;
	size_t mmap_sz;

	rb_map = container_of(map, struct bpf_ringbuf_map, map);
	mmap_sz = bpf_ringbuf_mmap_page_cnt(rb_map->rb) << PAGE_SHIFT;

	if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > mmap_sz)
		return -EINVAL;

	return remap_vmalloc_range(vma, rb_map->rb,
				   vma->vm_pgoff + RINGBUF_PGOFF);
}


			     struct timespec64 *ts, enum hrtimer_mode mode)
	__must_hold(&ctx->completion_lock)
{
	struct io_kiocb *req = io_timeout_extract(ctx, user_data);
	struct io_timeout_data *data;

	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout.off = 0; /* noseq */
	data = req->async_data;
	list_add_tail(&req->timeout.list, &ctx->timeout_list);
	hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
	data->timer.function = io_timeout_fn;
	hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
	return 0;


static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
				 struct io_rsrc_node *node, void *rsrc)
{
	struct io_rsrc_put *prsrc;

	prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
	if (!prsrc)
		return -ENOMEM;

	prsrc->tag = data->tags[idx];
	prsrc->rsrc = rsrc;
	list_add(&prsrc->list, &node->rsrc_list);
	return 0;


static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
				     long res, unsigned int cflags)
{
	struct io_overflow_cqe *ocqe;

	ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
	if (!ocqe) {
		/*
		 * If we're in ring overflow flush mode, or in task cancel mode,
		 * or cannot allocate an overflow entry, then we need to drop it
		 * on the floor.
		 */
		WRITE_ONCE(ctx->rings->cq_overflow, ++ctx->cached_cq_overflow);
		return false;
	}
	if (list_empty(&ctx->cq_overflow_list)) {
		set_bit(0, &ctx->sq_check_overflow);
		set_bit(0, &ctx->cq_check_overflow);
		ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
	}
	ocqe->cqe.user_data = user_data;
	ocqe->cqe.res = res;
	ocqe->cqe.flags = cflags;
	list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
	return true;
}


static void io_queue_sqe(struct io_kiocb *req)
{
	int ret;

	ret = io_req_defer(req);
	if (ret) {
		if (ret != -EIOCBQUEUED) {
fail_req:
			io_req_complete_failed(req, ret);
		}
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
		ret = io_req_prep_async(req);
		if (unlikely(ret))
			goto fail_req;
		io_queue_async_work(req);
	} else {
		__io_queue_sqe(req);
	}


static void io_clean_op(struct io_kiocb *req)
{
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
			kfree((void *)(unsigned long)req->rw.addr);
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_RECV:
			kfree(req->sr_msg.kbuf);
			break;
		}
		req->flags &= ~REQ_F_BUFFER_SELECTED;
	}

	if (req->flags & REQ_F_NEED_CLEANUP) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
		case IORING_OP_WRITE: {
			struct io_async_rw *io = req->async_data;
			if (io->free_iovec)
				kfree(io->free_iovec);
			break;
			}
		case IORING_OP_RECVMSG:
		case IORING_OP_SENDMSG: {
			struct io_async_msghdr *io = req->async_data;

			kfree(io->free_iov);
			break;
			}
		case IORING_OP_SPLICE:
		case IORING_OP_TEE:
			if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
				io_put_file(req->splice.file_in);
			break;
		case IORING_OP_OPENAT:
		case IORING_OP_OPENAT2:
			if (req->open.filename)
				putname(req->open.filename);
			break;
		case IORING_OP_RENAMEAT:
			putname(req->rename.oldpath);
			putname(req->rename.newpath);
			break;
		case IORING_OP_UNLINKAT:
			putname(req->unlink.filename);
			break;
		}
		req->flags &= ~REQ_F_NEED_CLEANUP;
	}
	if ((req->flags & REQ_F_POLLED) && req->apoll) {
		kfree(req->apoll->double_poll);
		kfree(req->apoll);
		req->apoll = NULL;
	}
	if (req->flags & REQ_F_INFLIGHT) {
		struct io_uring_task *tctx = req->task->io_uring;

		atomic_dec(&tctx->inflight_tracked);
		req->flags &= ~REQ_F_INFLIGHT;
	}


static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
{
	if (tracked)
		return atomic_read(&tctx->inflight_tracked);
	return percpu_counter_sum(&tctx->inflight);


static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	u64 sqe_addr = req->cancel.addr;
	struct io_tctx_node *node;
	int ret;

	/* tasks should wait for their io-wq threads, so safe w/o sync */
	ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
	spin_lock_irq(&ctx->completion_lock);
	if (ret != -ENOENT)
		goto done;
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr, false);
	if (ret != -ENOENT)
		goto done;
	spin_unlock_irq(&ctx->completion_lock);

	/* slow path, try all io-wq's */
	io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
	ret = -ENOENT;
	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
		struct io_uring_task *tctx = node->task->io_uring;

		ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
		if (ret != -ENOENT)
			break;
	}
	io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));

	spin_lock_irq(&ctx->completion_lock);
done:
	io_cqring_fill_event(ctx, req->user_data, ret, 0);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
	io_cqring_ev_posted(ctx);

	if (ret < 0)
		req_set_fail_links(req);
	io_put_req(req);
	return 0;


static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
{
	struct io_rsrc_node *ref_node;

	ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
	if (!ref_node)
		return NULL;

	if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
			    0, GFP_KERNEL)) {
		kfree(ref_node);
		return NULL;
	}
	INIT_LIST_HEAD(&ref_node->node);
	INIT_LIST_HEAD(&ref_node->rsrc_list);
	ref_node->done = false;
	return ref_node;


static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
	io_sq_thread_finish(ctx);

	if (ctx->mm_account) {
		mmdrop(ctx->mm_account);
		ctx->mm_account = NULL;
	}

	mutex_lock(&ctx->uring_lock);
	if (io_wait_rsrc_data(ctx->buf_data))
		__io_sqe_buffers_unregister(ctx);
	if (io_wait_rsrc_data(ctx->file_data))
		__io_sqe_files_unregister(ctx);
	if (ctx->rings)
		__io_cqring_overflow_flush(ctx, true);
	mutex_unlock(&ctx->uring_lock);
	io_eventfd_unregister(ctx);
	io_destroy_buffers(ctx);
	if (ctx->sq_creds)
		put_cred(ctx->sq_creds);

	/* there are no registered resources left, nobody uses it */
	if (ctx->rsrc_node)
		io_rsrc_node_destroy(ctx->rsrc_node);
	if (ctx->rsrc_backup_node)
		io_rsrc_node_destroy(ctx->rsrc_backup_node);
	flush_delayed_work(&ctx->rsrc_put_work);

	WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
	WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));

#if defined(CONFIG_UNIX)
	if (ctx->ring_sock) {
		ctx->ring_sock->file = NULL; /* so that iput() is called */
		sock_release(ctx->ring_sock);
	}
#endif

	io_mem_free(ctx->rings);
	io_mem_free(ctx->sq_sqes);

	percpu_ref_exit(&ctx->refs);
	free_uid(ctx->user);
	io_req_caches_free(ctx);
	if (ctx->hash_map)
		io_wq_put_hash(ctx->hash_map);
	kfree(ctx->cancel_hash);
	kfree(ctx->dummy_ubuf);
	kfree(ctx);


static bool io_wait_rsrc_data(struct io_rsrc_data *data)
{
	if (!data)
		return false;
	if (!atomic_dec_and_test(&data->refs))
		wait_for_completion(&data->done);
	return true;


			  bool poll_only)
	__must_hold(&ctx->completion_lock)
{
	struct io_kiocb *req;

	req = io_poll_find(ctx, sqe_addr, poll_only);
	if (!req)
		return -ENOENT;
	if (io_poll_remove_one(req))
		return 0;

	return -EALREADY;


static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_poll_iocb *poll = &req->poll;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_poll_table ipt;
	__poll_t mask;

	ipt.pt._qproc = io_poll_queue_proc;

	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
					io_poll_wake);

	if (mask) { /* no async, we'd stolen it */
		ipt.error = 0;
		io_poll_complete(req, mask);
	}
	spin_unlock_irq(&ctx->completion_lock);

	if (mask) {
		io_cqring_ev_posted(ctx);
		if (poll->events & EPOLLONESHOT)
			io_put_req(req);
	}
	return ipt.error;


static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
			     int sync, void *arg)
{
	struct wait_page_queue *wpq;
	struct io_kiocb *req = wait->private;
	struct wait_page_key *key = arg;

	wpq = container_of(wait, struct wait_page_queue, wait);

	if (!wake_page_match(wpq, key))
		return 0;

	req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
	list_del_init(&wait->entry);

	/* submit ref gets dropped, acquire a new one */
	req_ref_get(req);
	io_req_task_queue(req);
	return 1;
}


static bool io_arm_poll_handler(struct io_kiocb *req)
{
	const struct io_op_def *def = &io_op_defs[req->opcode];
	struct io_ring_ctx *ctx = req->ctx;
	struct async_poll *apoll;
	struct io_poll_table ipt;
	__poll_t mask, ret;
	int rw;

	if (!req->file || !file_can_poll(req->file))
		return false;
	if (req->flags & REQ_F_POLLED)
		return false;
	if (def->pollin)
		rw = READ;
	else if (def->pollout)
		rw = WRITE;
	else
		return false;
	/* if we can't nonblock try, then no point in arming a poll handler */
	if (!io_file_supports_async(req, rw))
		return false;

	apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
	if (unlikely(!apoll))
		return false;
	apoll->double_poll = NULL;

	req->flags |= REQ_F_POLLED;
	req->apoll = apoll;

	mask = EPOLLONESHOT;
	if (def->pollin)
		mask |= POLLIN | POLLRDNORM;
	if (def->pollout)
		mask |= POLLOUT | POLLWRNORM;

	/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
	if ((req->opcode == IORING_OP_RECVMSG) &&
	    (req->sr_msg.msg_flags & MSG_ERRQUEUE))
		mask &= ~POLLIN;

	mask |= POLLERR | POLLPRI;

	ipt.pt._qproc = io_async_queue_proc;

	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
					io_async_wake);
	if (ret || ipt.error) {
		io_poll_remove_double(req);
		spin_unlock_irq(&ctx->completion_lock);
		return false;
	}
	spin_unlock_irq(&ctx->completion_lock);
	trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
					apoll->poll.events);
	return true;


static void io_ring_exit_work(struct work_struct *work)
{
	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
	unsigned long timeout = jiffies + HZ * 60 * 5;
	struct io_tctx_exit exit;
	struct io_tctx_node *node;
	int ret;

	/*
	 * If we're doing polled IO and end up having requests being
	 * submitted async (out-of-line), then completions can come in while
	 * we're waiting for refs to drop. We need to reap these manually,
	 * as nobody else will be looking for them.
	 */
	do {
		io_uring_try_cancel_requests(ctx, NULL, NULL);
		if (ctx->sq_data) {
			struct io_sq_data *sqd = ctx->sq_data;
			struct task_struct *tsk;

			io_sq_thread_park(sqd);
			tsk = sqd->thread;
			if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
				io_wq_cancel_cb(tsk->io_uring->io_wq,
						io_cancel_ctx_cb, ctx, true);
			io_sq_thread_unpark(sqd);
		}

		WARN_ON_ONCE(time_after(jiffies, timeout));
	} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));

	init_completion(&exit.completion);
	init_task_work(&exit.task_work, io_tctx_exit_cb);
	exit.ctx = ctx;
	/*
	 * Some may use context even when all refs and requests have been put,
	 * and they are free to do so while still holding uring_lock or
	 * completion_lock, see __io_req_task_submit(). Apart from other work,
	 * this lock/unlock section also waits them to finish.
	 */
	mutex_lock(&ctx->uring_lock);
	while (!list_empty(&ctx->tctx_list)) {
		WARN_ON_ONCE(time_after(jiffies, timeout));

		node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
					ctx_node);
		/* don't spin on a single task if cancellation failed */
		list_rotate_left(&ctx->tctx_list);
		ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
		if (WARN_ON_ONCE(ret))
			continue;
		wake_up_process(node->task);

		mutex_unlock(&ctx->uring_lock);
		wait_for_completion(&exit.completion);
		mutex_lock(&ctx->uring_lock);
	}
	mutex_unlock(&ctx->uring_lock);
	spin_lock_irq(&ctx->completion_lock);
	spin_unlock_irq(&ctx->completion_lock);

	io_ring_ctx_free(ctx);


	__must_hold(&req->ctx->completion_lock)
{
	bool posted = false;

	if (likely(req->flags & REQ_F_LINK_TIMEOUT))
		posted = io_kill_linked_timeout(req);
	if (unlikely((req->flags & REQ_F_FAIL_LINK) &&
		     !(req->flags & REQ_F_HARDLINK))) {
		posted |= (req->link != NULL);
		io_fail_links(req);
	}
	return posted;
}


void blk_start_plug(struct blk_plug *plug)
{
	struct task_struct *tsk = current;

	/*
	 * If this is a nested plug, don't actually assign it.
	 */
	if (tsk->plug)
		return;

	INIT_LIST_HEAD(&plug->mq_list);
	INIT_LIST_HEAD(&plug->cb_list);
	plug->rq_count = 0;
	plug->multiple_queues = false;

	/*
	 * Store ordering should not be needed here, since a potential
	 * preempt will imply a full memory barrier
	 */
	tsk->plug = plug;
}


static inline struct io_context *create_io_context(gfp_t gfp_mask, int node)
{
	WARN_ON_ONCE(irqs_disabled());
	if (unlikely(!current->io_context))
		create_task_io_context(current, gfp_mask, node);
	return current->io_context;
}


static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
{
	return queue_var_show(blk_queue_nr_zones(q), page);
}


int elv_register_queue(struct request_queue *q)
{
	struct elevator_queue *e = q->elevator;
	int error;

	lockdep_assert_held(&q->sysfs_lock);

	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
	if (!error) {
		struct elv_fs_entry *attr = e->type->elevator_attrs;
		if (attr) {
			while (attr->attr.name) {
				if (sysfs_create_file(&e->kobj, &attr->attr))
					break;
				attr++;
			}
		}
		kobject_uevent(&e->kobj, KOBJ_ADD);
		e->registered = 1;
	}
	return error;
}


static void print_req_error(struct request *req, blk_status_t status)
{
	int idx = (__force int)status;

	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
		return;

	printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu flags %x\n",
				__func__, blk_errors[idx].name,
				req->rq_disk ?  req->rq_disk->disk_name : "?",
				(unsigned long long)blk_rq_pos(req),
				req->cmd_flags);
}


blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
{
	if (blk_cloned_rq_check_limits(q, rq))
		return BLK_STS_IOERR;

	if (rq->rq_disk &&
	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
		return BLK_STS_IOERR;

	if (blk_queue_io_stat(q))
		blk_account_io_start(rq, true);

	/*
	 * Since we have a scheduler attached on the top device,
	 * bypass a potential scheduler on the bottom device for
	 * insert.
	 */
	return blk_mq_request_issue_directly(rq, true);
}


void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
{
	__elv_rqhash_del(rq);
	elv_rqhash_add(q, rq);
}


queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
{
	ssize_t ret = -EINVAL;
#ifdef CONFIG_SMP
	unsigned long val;

	ret = queue_var_store(&val, page, count);
	if (ret < 0)
		return ret;

	if (val == 2) {
		blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
		blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 1) {
		blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
		blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	} else if (val == 0) {
		blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
		blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
	}
#endif
	return ret;
}


static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
				  size_t count)
{
	struct rq_qos *rqos;
	ssize_t ret;
	s64 val;

	ret = queue_var_store64(&val, page);
	if (ret < 0)
		return ret;
	if (val < -1)
		return -EINVAL;

	rqos = wbt_rq_qos(q);
	if (!rqos) {
		ret = wbt_init(q);
		if (ret)
			return ret;
	}

	if (val == -1)
		val = wbt_default_latency_nsec(q);
	else if (val >= 0)
		val *= 1000ULL;

	if (wbt_get_min_lat(q) == val)
		return count;

	/*
	 * Ensure that the queue is idled, in case the latency update
	 * ends up either enabling or disabling wbt completely. We can't
	 * have IO inflight if that happens.
	 */
	blk_mq_freeze_queue(q);
	blk_mq_quiesce_queue(q);

	wbt_set_min_lat(q, val);
	wbt_update_limits(q);

	blk_mq_unquiesce_queue(q);
	blk_mq_unfreeze_queue(q);

	return count;
}


void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
{
	mutex_lock(&e->sysfs_lock);
	if (e->type->ops.exit_sched)
		blk_mq_exit_sched(q, e);
	mutex_unlock(&e->sysfs_lock);

	kobject_put(&e->kobj);
}


bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
		struct bio *bio)
{
	unsigned short segments = blk_rq_nr_discard_segments(req);

	if (segments >= queue_max_discard_segments(q))
		goto no_merge;
	if (blk_rq_sectors(req) + bio_sectors(bio) >
	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
		goto no_merge;

	req->biotail->bi_next = bio;
	req->biotail = bio;
	req->__data_len += bio->bi_iter.bi_size;
	req->nr_phys_segments = segments + 1;

	blk_account_io_start(req, false);
	return true;
no_merge:
	req_set_nomerge(q, req);
	return false;
}


int elv_register(struct elevator_type *e)
{
	/* create icq_cache if requested */
	if (e->icq_size) {
		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
			return -EINVAL;

		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
			 "%s_io_cq", e->elevator_name);
		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
						 e->icq_align, 0, NULL);
		if (!e->icq_cache)
			return -ENOMEM;
	}

	/* register, don't allow duplicate names */
	spin_lock(&elv_list_lock);
	if (elevator_find(e->elevator_name)) {
		spin_unlock(&elv_list_lock);
		kmem_cache_destroy(e->icq_cache);
		return -EBUSY;
	}
	list_add_tail(&e->list, &elv_list);
	spin_unlock(&elv_list_lock);

	printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);

	return 0;
}


static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
				   struct blk_mq_hw_ctx *hctx,
				   unsigned int hctx_idx)
{
	if (hctx->sched_tags) {
		blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
		blk_mq_free_rq_map(hctx->sched_tags);
		hctx->sched_tags = NULL;
	}
}


static inline bool blk_do_io_stat(struct request *rq)
{
	return rq->rq_disk &&
	       (rq->rq_flags & RQF_IO_STAT) &&
		!blk_rq_is_passthrough(rq);
}


void blk_put_request(struct request *req)
{
	blk_mq_free_request(req);
}


void elv_rqhash_del(struct request_queue *q, struct request *rq)
{
	if (ELV_ON_HASH(rq))
		__elv_rqhash_del(rq);
}


void blk_sync_queue(struct request_queue *q)
{
	del_timer_sync(&q->timeout);
	cancel_work_sync(&q->timeout_work);
}


struct elevator_queue *elevator_alloc(struct request_queue *q,
				  struct elevator_type *e)
{
	struct elevator_queue *eq;

	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
	if (unlikely(!eq))
		return NULL;

	eq->type = e;
	kobject_init(&eq->kobj, &elv_ktype);
	mutex_init(&eq->sysfs_lock);
	hash_init(eq->hash);

	return eq;
}


static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
{
}


static int cipso_v4_parsetag_rng(const struct cipso_v4_doi *doi_def,
				 const unsigned char *tag,
				 struct netlbl_lsm_secattr *secattr)
{
	int ret_val;
	u8 tag_len = tag[1];
	u32 level;

	ret_val = cipso_v4_map_lvl_ntoh(doi_def, tag[3], &level);
	if (ret_val != 0)
		return ret_val;
	secattr->attr.mls.lvl = level;
	secattr->flags |= NETLBL_SECATTR_MLS_LVL;

	if (tag_len > 4) {
		ret_val = cipso_v4_map_cat_rng_ntoh(doi_def,
						    &tag[4],
						    tag_len - 4,
						    secattr);
		if (ret_val != 0) {
			netlbl_catmap_free(secattr->attr.mls.cat);
			return ret_val;
		}

		if (secattr->attr.mls.cat)
			secattr->flags |= NETLBL_SECATTR_MLS_CAT;
	}

	return 0;
}


void cipso_v4_doi_putdef(struct cipso_v4_doi *doi_def)
{
	if (!doi_def)
		return;

	if (!refcount_dec_and_test(&doi_def->refcount))
		return;

	cipso_v4_cache_invalidate();
	call_rcu(&doi_def->rcu, cipso_v4_doi_free_rcu);
}


static int netlbl_cipsov4_remove_cb(struct netlbl_dom_map *entry, void *arg)
{
	struct netlbl_domhsh_walk_arg *cb_arg = arg;

	if (entry->def.type == NETLBL_NLTYPE_CIPSOV4 &&
	    entry->def.cipso->doi == cb_arg->doi)
		return netlbl_domhsh_remove_entry(entry, cb_arg->audit_info);

	return 0;
}


static int calipso_opt_del(struct ipv6_opt_hdr *hop,
			   struct ipv6_opt_hdr **new)
{
	int ret_val;
	unsigned int start, end, delta, pad, hop_len;

	ret_val = calipso_opt_find(hop, &start, &end);
	if (ret_val)
		return ret_val;

	hop_len = ipv6_optlen(hop);
	if (start == sizeof(*hop) && end == hop_len) {
		/* There's no other option in the header so return NULL */
		*new = NULL;
		return 0;
	}

	delta = (end - start) & ~7;
	*new = kzalloc(hop_len - delta, GFP_ATOMIC);
	if (!*new)
		return -ENOMEM;

	memcpy(*new, hop, start);
	(*new)->hdrlen -= delta / 8;
	pad = (end - start) & 7;
	calipso_pad_write((unsigned char *)*new, start, pad);
	if (end != hop_len)
		memcpy((char *)*new + start + pad, (char *)hop + end,
		       hop_len - end);

	return 0;
}


static int calipso_tlv_len(struct ipv6_opt_hdr *opt, unsigned int offset)
{
	unsigned char *tlv = (unsigned char *)opt;
	unsigned int opt_len = ipv6_optlen(opt), tlv_len;

	if (offset < sizeof(*opt) || offset >= opt_len)
		return -EINVAL;
	if (tlv[offset] == IPV6_TLV_PAD1)
		return 1;
	if (offset + 1 >= opt_len)
		return -EINVAL;
	tlv_len = tlv[offset + 1] + 2;
	if (offset + tlv_len > opt_len)
		return -EINVAL;
	return tlv_len;
}


static int netlbl_cipsov4_listall(struct sk_buff *skb,
				  struct netlink_callback *cb)
{
	struct netlbl_cipsov4_doiwalk_arg cb_arg;
	u32 doi_skip = cb->args[0];

	cb_arg.nl_cb = cb;
	cb_arg.skb = skb;
	cb_arg.seq = cb->nlh->nlmsg_seq;

	cipso_v4_doi_walk(&doi_skip, netlbl_cipsov4_listall_cb, &cb_arg);

	cb->args[0] = doi_skip;
	return skb->len;
}


int mac802154_llsec_devkey_del(struct mac802154_llsec *sec,
			       __le64 dev_addr,
			       const struct ieee802154_llsec_device_key *key)
{
	struct mac802154_llsec_device *dev;
	struct mac802154_llsec_device_key *devkey;

	dev = llsec_dev_find_long(sec, dev_addr);

	if (!dev)
		return -ENOENT;

	devkey = llsec_devkey_find(dev, &key->key_id);
	if (!devkey)
		return -ENOENT;

	list_del_rcu(&devkey->devkey.list);
	kfree_rcu(devkey, rcu);
	return 0;
}


static int llsec_recover_addr(struct mac802154_llsec *sec,
			      struct ieee802154_addr *addr)
{
	__le16 caddr = sec->params.coord_shortaddr;

	addr->pan_id = sec->params.pan_id;

	if (caddr == cpu_to_le16(IEEE802154_ADDR_BROADCAST)) {
		return -EINVAL;
	} else if (caddr == cpu_to_le16(IEEE802154_ADDR_UNDEF)) {
		addr->extended_addr = sec->params.coord_hwaddr;
		addr->mode = IEEE802154_ADDR_LONG;
	} else {
		addr->short_addr = sec->params.coord_shortaddr;
		addr->mode = IEEE802154_ADDR_SHORT;
	}

	return 0;
}


int mac802154_llsec_get_params(struct mac802154_llsec *sec,
			       struct ieee802154_llsec_params *params)
{
	read_lock_bh(&sec->lock);
	*params = sec->params;
	read_unlock_bh(&sec->lock);

	return 0;
}


static void llsec_geniv(u8 iv[16], __le64 addr,
			const struct ieee802154_sechdr *sec)
{
	__be64 addr_bytes = (__force __be64) swab64((__force u64) addr);
	__be32 frame_counter = (__force __be32) swab32((__force u32) sec->frame_counter);

	iv[0] = 1; /* L' = L - 1 = 1 */
	memcpy(iv + 1, &addr_bytes, sizeof(addr_bytes));
	memcpy(iv + 9, &frame_counter, sizeof(frame_counter));
	iv[13] = sec->level;
	iv[14] = 0;
	iv[15] = 1;
}


static bool llsec_dev_use_shortaddr(__le16 short_addr)
{
	return short_addr != cpu_to_le16(IEEE802154_ADDR_UNDEF) &&
		short_addr != cpu_to_le16(0xffff);
}


void mac802154_llsec_init(struct mac802154_llsec *sec)
{
	memset(sec, 0, sizeof(*sec));

	memset(&sec->params.default_key_source, 0xFF, IEEE802154_ADDR_LEN);

	INIT_LIST_HEAD(&sec->table.security_levels);
	INIT_LIST_HEAD(&sec->table.devices);
	INIT_LIST_HEAD(&sec->table.keys);
	hash_init(sec->devices_short);
	hash_init(sec->devices_hw);
	rwlock_init(&sec->lock);
}


void hci_event_packet(struct hci_dev *hdev, struct sk_buff *skb)
{
	struct hci_event_hdr *hdr = (void *) skb->data;
	hci_req_complete_t req_complete = NULL;
	hci_req_complete_skb_t req_complete_skb = NULL;
	struct sk_buff *orig_skb = NULL;
	u8 status = 0, event = hdr->evt, req_evt = 0;
	u16 opcode = HCI_OP_NOP;

	if (!event) {
		bt_dev_warn(hdev, "Received unexpected HCI Event 00000000");
		goto done;
	}

	if (hdev->sent_cmd && bt_cb(hdev->sent_cmd)->hci.req_event == event) {
		struct hci_command_hdr *cmd_hdr = (void *) hdev->sent_cmd->data;
		opcode = __le16_to_cpu(cmd_hdr->opcode);
		hci_req_cmd_complete(hdev, opcode, status, &req_complete,
				     &req_complete_skb);
		req_evt = event;
	}

	/* If it looks like we might end up having to call
	 * req_complete_skb, store a pristine copy of the skb since the
	 * various handlers may modify the original one through
	 * skb_pull() calls, etc.
	 */
	if (req_complete_skb || event == HCI_EV_CMD_STATUS ||
	    event == HCI_EV_CMD_COMPLETE)
		orig_skb = skb_clone(skb, GFP_KERNEL);

	skb_pull(skb, HCI_EVENT_HDR_SIZE);

	/* Store wake reason if we're suspended */
	hci_store_wake_reason(hdev, event, skb);

	switch (event) {
	case HCI_EV_INQUIRY_COMPLETE:
		hci_inquiry_complete_evt(hdev, skb);
		break;

	case HCI_EV_INQUIRY_RESULT:
		hci_inquiry_result_evt(hdev, skb);
		break;

	case HCI_EV_CONN_COMPLETE:
		hci_conn_complete_evt(hdev, skb);
		break;

	case HCI_EV_CONN_REQUEST:
		hci_conn_request_evt(hdev, skb);
		break;

	case HCI_EV_DISCONN_COMPLETE:
		hci_disconn_complete_evt(hdev, skb);
		break;

	case HCI_EV_AUTH_COMPLETE:
		hci_auth_complete_evt(hdev, skb);
		break;

	case HCI_EV_REMOTE_NAME:
		hci_remote_name_evt(hdev, skb);
		break;

	case HCI_EV_ENCRYPT_CHANGE:
		hci_encrypt_change_evt(hdev, skb);
		break;

	case HCI_EV_CHANGE_LINK_KEY_COMPLETE:
		hci_change_link_key_complete_evt(hdev, skb);
		break;

	case HCI_EV_REMOTE_FEATURES:
		hci_remote_features_evt(hdev, skb);
		break;

	case HCI_EV_CMD_COMPLETE:
		hci_cmd_complete_evt(hdev, skb, &opcode, &status,
				     &req_complete, &req_complete_skb);
		break;

	case HCI_EV_CMD_STATUS:
		hci_cmd_status_evt(hdev, skb, &opcode, &status, &req_complete,
				   &req_complete_skb);
		break;

	case HCI_EV_HARDWARE_ERROR:
		hci_hardware_error_evt(hdev, skb);
		break;

	case HCI_EV_ROLE_CHANGE:
		hci_role_change_evt(hdev, skb);
		break;

	case HCI_EV_NUM_COMP_PKTS:
		hci_num_comp_pkts_evt(hdev, skb);
		break;

	case HCI_EV_MODE_CHANGE:
		hci_mode_change_evt(hdev, skb);
		break;

	case HCI_EV_PIN_CODE_REQ:
		hci_pin_code_request_evt(hdev, skb);
		break;

	case HCI_EV_LINK_KEY_REQ:
		hci_link_key_request_evt(hdev, skb);
		break;

	case HCI_EV_LINK_KEY_NOTIFY:
		hci_link_key_notify_evt(hdev, skb);
		break;

	case HCI_EV_CLOCK_OFFSET:
		hci_clock_offset_evt(hdev, skb);
		break;

	case HCI_EV_PKT_TYPE_CHANGE:
		hci_pkt_type_change_evt(hdev, skb);
		break;

	case HCI_EV_PSCAN_REP_MODE:
		hci_pscan_rep_mode_evt(hdev, skb);
		break;

	case HCI_EV_INQUIRY_RESULT_WITH_RSSI:
		hci_inquiry_result_with_rssi_evt(hdev, skb);
		break;

	case HCI_EV_REMOTE_EXT_FEATURES:
		hci_remote_ext_features_evt(hdev, skb);
		break;

	case HCI_EV_SYNC_CONN_COMPLETE:
		hci_sync_conn_complete_evt(hdev, skb);
		break;

	case HCI_EV_EXTENDED_INQUIRY_RESULT:
		hci_extended_inquiry_result_evt(hdev, skb);
		break;

	case HCI_EV_KEY_REFRESH_COMPLETE:
		hci_key_refresh_complete_evt(hdev, skb);
		break;

	case HCI_EV_IO_CAPA_REQUEST:
		hci_io_capa_request_evt(hdev, skb);
		break;

	case HCI_EV_IO_CAPA_REPLY:
		hci_io_capa_reply_evt(hdev, skb);
		break;

	case HCI_EV_USER_CONFIRM_REQUEST:
		hci_user_confirm_request_evt(hdev, skb);
		break;

	case HCI_EV_USER_PASSKEY_REQUEST:
		hci_user_passkey_request_evt(hdev, skb);
		break;

	case HCI_EV_USER_PASSKEY_NOTIFY:
		hci_user_passkey_notify_evt(hdev, skb);
		break;

	case HCI_EV_KEYPRESS_NOTIFY:
		hci_keypress_notify_evt(hdev, skb);
		break;

	case HCI_EV_SIMPLE_PAIR_COMPLETE:
		hci_simple_pair_complete_evt(hdev, skb);
		break;

	case HCI_EV_REMOTE_HOST_FEATURES:
		hci_remote_host_features_evt(hdev, skb);
		break;

	case HCI_EV_LE_META:
		hci_le_meta_evt(hdev, skb);
		break;

	case HCI_EV_REMOTE_OOB_DATA_REQUEST:
		hci_remote_oob_data_request_evt(hdev, skb);
		break;

#if IS_ENABLED(CONFIG_BT_HS)
	case HCI_EV_CHANNEL_SELECTED:
		hci_chan_selected_evt(hdev, skb);
		break;

	case HCI_EV_PHY_LINK_COMPLETE:
		hci_phy_link_complete_evt(hdev, skb);
		break;

	case HCI_EV_LOGICAL_LINK_COMPLETE:
		hci_loglink_complete_evt(hdev, skb);
		break;

	case HCI_EV_DISCONN_LOGICAL_LINK_COMPLETE:
		hci_disconn_loglink_complete_evt(hdev, skb);
		break;

	case HCI_EV_DISCONN_PHY_LINK_COMPLETE:
		hci_disconn_phylink_complete_evt(hdev, skb);
		break;
#endif

	case HCI_EV_NUM_COMP_BLOCKS:
		hci_num_comp_blocks_evt(hdev, skb);
		break;

	case HCI_EV_VENDOR:
		msft_vendor_evt(hdev, skb);
		break;

	default:
		BT_DBG("%s event 0x%2.2x", hdev->name, event);
		break;
	}

	if (req_complete) {
		req_complete(hdev, status, opcode);
	} else if (req_complete_skb) {
		if (!hci_get_cmd_complete(hdev, opcode, req_evt, orig_skb)) {
			kfree_skb(orig_skb);
			orig_skb = NULL;
		}
		req_complete_skb(hdev, status, opcode, orig_skb);
	}

done:
	kfree_skb(orig_skb);
	kfree_skb(skb);
	hdev->stat.evt_rx++;
}


static void cs_le_create_conn(struct hci_dev *hdev, bdaddr_t *peer_addr,
			      u8 peer_addr_type, u8 own_address_type,
			      u8 filter_policy)
{
	struct hci_conn *conn;

	conn = hci_conn_hash_lookup_le(hdev, peer_addr,
				       peer_addr_type);
	if (!conn)
		return;

	/* When using controller based address resolution, then the new
	 * address types 0x02 and 0x03 are used. These types need to be
	 * converted back into either public address or random address type
	 */
	if (use_ll_privacy(hdev) &&
	    hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) {
		switch (own_address_type) {
		case ADDR_LE_DEV_PUBLIC_RESOLVED:
			own_address_type = ADDR_LE_DEV_PUBLIC;
			break;
		case ADDR_LE_DEV_RANDOM_RESOLVED:
			own_address_type = ADDR_LE_DEV_RANDOM;
			break;
		}
	}

	/* Store the initiator and responder address information which
	 * is needed for SMP. These values will not change during the
	 * lifetime of the connection.
	 */
	conn->init_addr_type = own_address_type;
	if (own_address_type == ADDR_LE_DEV_RANDOM)
		bacpy(&conn->init_addr, &hdev->random_addr);
	else
		bacpy(&conn->init_addr, &hdev->bdaddr);

	conn->resp_addr_type = peer_addr_type;
	bacpy(&conn->resp_addr, peer_addr);

	/* We don't want the connection attempt to stick around
	 * indefinitely since LE doesn't have a page timeout concept
	 * like BR/EDR. Set a timer for any connection that doesn't use
	 * the white list for connecting.
	 */
	if (filter_policy == HCI_LE_USE_PEER_ADDR)
		queue_delayed_work(conn->hdev->workqueue,
				   &conn->le_conn_timeout,
				   conn->conn_timeout);
}


polkit_system_bus_name_class_init (PolkitSystemBusNameClass *klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);

  gobject_class->get_property = polkit_system_bus_name_get_property;
  gobject_class->set_property = polkit_system_bus_name_set_property;
  gobject_class->finalize     = polkit_system_bus_name_finalize;

  /**
   * PolkitSystemBusName:name:
   *
   * The unique name on the system message bus.
   */
  g_object_class_install_property (gobject_class,
                                   PROP_NAME,
                                   g_param_spec_string ("name",
                                                        "Name",
                                                        "The unique name on the system message bus",
                                                        NULL,
                                                        G_PARAM_CONSTRUCT |
                                                        G_PARAM_READWRITE |
                                                        G_PARAM_STATIC_NAME |
                                                        G_PARAM_STATIC_BLURB |
                                                        G_PARAM_STATIC_NICK));

}


polkit_system_bus_name_exists_finish (PolkitSubject  *subject,
                                      GAsyncResult   *res,
                                      GError        **error)
{
  GSimpleAsyncResult *simple = G_SIMPLE_ASYNC_RESULT (res);
  gboolean ret;

  g_warn_if_fail (g_simple_async_result_get_source_tag (simple) == polkit_system_bus_name_exists);

  ret = FALSE;

  if (g_simple_async_result_propagate_error (simple, error))
    goto out;

  ret = g_simple_async_result_get_op_res_gboolean (simple);

 out:
  return ret;
}


polkit_system_bus_name_set_name (PolkitSystemBusName *system_bus_name,
                                 const gchar         *name)
{
  g_return_if_fail (POLKIT_IS_SYSTEM_BUS_NAME (system_bus_name));
  g_return_if_fail (g_dbus_is_unique_name (name));
  g_free (system_bus_name->name);
  system_bus_name->name = g_strdup (name);
}


_XF86BigfontFreeCodes (
    XExtData *extension)
{
    /* Don't Xfree(extension->private_data) because it is on the same malloc
       chunk as extension. */
    /* Don't Xfree(extension->private_data->codes) because this is shared with
       the display's ext_procs list. */
    return 0;
}


urnHandleReply(void *data, StoreIOBuffer result)
{
    UrnState *urnState = static_cast<UrnState *>(data);
    StoreEntry *e = urnState->entry;
    StoreEntry *urlres_e = urnState->urlres_e;
    char *s = NULL;
    size_t k;
    HttpReply *rep;
    url_entry *urls;
    url_entry *u;
    url_entry *min_u;
    ErrorState *err;
    int i;
    int urlcnt = 0;
    char *buf = urnState->reqbuf;
    StoreIOBuffer tempBuffer;

    debugs(52, 3, "urnHandleReply: Called with size=" << result.length << ".");

    if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.flags.error) {
        delete urnState;
        return;
    }

    /* Update reqofs to point to where in the buffer we'd be */
    urnState->reqofs += result.length;

    /* Handle reqofs being bigger than normal */
    if (urnState->reqofs >= URN_REQBUF_SZ) {
        delete urnState;
        return;
    }

    /* If we haven't received the entire object (urn), copy more */
    if (urlres_e->store_status == STORE_PENDING) {
        Must(result.length > 0); // zero length ought to imply STORE_OK
        tempBuffer.offset = urnState->reqofs;
        tempBuffer.length = URN_REQBUF_SZ - urnState->reqofs;
        tempBuffer.data = urnState->reqbuf + urnState->reqofs;
        storeClientCopy(urnState->sc, urlres_e,
                        tempBuffer,
                        urnHandleReply,
                        urnState);
        return;
    }

    /* we know its STORE_OK */
    k = headersEnd(buf, urnState->reqofs);

    if (0 == k) {
        debugs(52, DBG_IMPORTANT, "urnHandleReply: didn't find end-of-headers for " << e->url()  );
        delete urnState;
        return;
    }

    s = buf + k;
    // TODO: Check whether we should parse urlres_e reply, as before 528b2c61.
    rep = new HttpReply;
    rep->parseCharBuf(buf, k);
    debugs(52, 3, "reply exists, code=" << rep->sline.status() << ".");

    if (rep->sline.status() != Http::scOkay) {
        debugs(52, 3, "urnHandleReply: failed.");
        err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
        err->url = xstrdup(e->url());
        errorAppendEntry(e, err);
        delete rep;
        delete urnState;
        return;
    }

    delete rep;

    while (xisspace(*s))
        ++s;

    urls = urnParseReply(s, urnState->request->method);

    if (!urls) {     /* unknown URN error */
        debugs(52, 3, "urnTranslateDone: unknown URN " << e->url());
        err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw(), urnState->ale);
        err->url = xstrdup(e->url());
        errorAppendEntry(e, err);
        delete urnState;
        return;
    }

    for (i = 0; urls[i].url; ++i)
        ++urlcnt;

    debugs(53, 3, "urnFindMinRtt: Counted " << i << " URLs");

    min_u = urnFindMinRtt(urls, urnState->request->method, NULL);
    qsort(urls, urlcnt, sizeof(*urls), url_entry_sort);
    e->buffer();
    SBuf body;
    SBuf *mb = &body; // diff reduction hack; TODO: Remove
    mb->appendf( "<TITLE>Select URL for %s</TITLE>\n"
                 "\n"
                 "
Select URL for %s
\n"
                 "
\n", e->url(), e->url());

    for (i = 0; i < urlcnt; ++i) {
        u = &urls[i];
        debugs(52, 3, "URL {" <url << "}");
        mb->appendf(
            "
%s	", u->url, u->url);

        if (urls[i].rtt > 0)
            mb->appendf(
                "%4d <it>ms</it>	", u->rtt);
        else
            mb->appendf("Unknown	");

        mb->appendf("%s
\n", u->flags.cached ? "    [cached]" : " ");
    }

    mb->appendf(
        "
"
        "
\n"
        "<ADDRESS>\n"
        "Generated by %s@%s\n"
        "</ADDRESS>\n",
        APP_FULLNAME, getMyHostname());
    rep = new HttpReply;
    rep->setHeaders(Http::scFound, NULL, "text/html", mb->length(), 0, squid_curtime);

    if (min_u) {
        rep->header.putStr(Http::HdrType::LOCATION, min_u->url);
    }

    rep->body.set(body);
    e->replaceHttpReply(rep);
    e->complete();

    for (i = 0; i < urlcnt; ++i) {
        safe_free(urls[i].url);
        safe_free(urls[i].host);
    }

    safe_free(urls);

    delete urnState;
}


Mgr::QueryParams::CreateParam(QueryParam::Type aType)
{
    switch (aType) {
    case QueryParam::ptInt:
        return new IntParam();

    case QueryParam::ptString:
        return new StringParam();

    default:
        throw TexcHere("unknown parameter type");
        break;
    }
    return NULL;
}


testCacheManager::testCreate()
{
    CacheManager::GetInstance(); //it's a singleton..
}


ParseParamValue(const SBuf &rawValue)
{
    static const CharacterSet comma("comma", ",");

    Parser::Tokenizer tok(rawValue);
    std::vector<int> array;
    int64_t intVal = 0;
    while (tok.int64(intVal, 10, false)) {
        Must(intVal >= std::numeric_limits<int>::min());
        Must(intVal <= std::numeric_limits<int>::max());
        array.emplace_back(intVal);
        // integer list has comma between values.
        // Require at least one potential DIGIT after the skipped ','
        if (tok.remaining().length() > 1)
            (void)tok.skipOne(comma);
    }

    if (tok.atEnd())
        return new Mgr::IntParam(array);
    else
        return new Mgr::StringParam(SBufToString(rawValue));
}


CacheManager::registerProfile(const Mgr::ActionProfile::Pointer &profile)
{
    Must(profile != NULL);
    if (!CacheManager::findAction(profile->name)) {
        menu_.push_back(profile);
        debugs(16, 3, HERE << "registered profile: " << *profile);
    } else {
        debugs(16, 2, HERE << "skipped duplicate profile: " << *profile);
    }
}


virNodeDevPCICapSRIOVVirtualParseXML(xmlXPathContextPtr ctxt,
                                     virNodeDevCapPCIDevPtr pci_dev)
{
    int ret = -1;
    xmlNodePtr *addresses = NULL;
    int naddresses = virXPathNodeSet("./address", ctxt, &addresses);
    char *maxFuncsStr = virXPathString("string(./@maxCount)", ctxt);
    size_t i;

    if (naddresses < 0)
        goto cleanup;

    if (maxFuncsStr &&
        virStrToLong_uip(maxFuncsStr, NULL, 10,
                         &pci_dev->max_virtual_functions) < 0) {
        virReportError(VIR_ERR_XML_ERROR, "%s",
                       _("Malformed 'maxCount' parameter"));
        goto cleanup;
    }

    pci_dev->virtual_functions = g_new0(virPCIDeviceAddressPtr, naddresses);

    for (i = 0; i < naddresses; i++) {
        g_autoptr(virPCIDeviceAddress) addr = NULL;

        addr = g_new0(virPCIDeviceAddress, 1);

        if (virPCIDeviceAddressParseXML(addresses[i], addr) < 0)
            goto cleanup;

        if (VIR_APPEND_ELEMENT(pci_dev->virtual_functions,
                               pci_dev->num_virtual_functions,
                               addr) < 0)
            goto cleanup;
    }

    pci_dev->flags |= VIR_NODE_DEV_CAP_FLAG_PCI_VIRTUAL_FUNCTION;
    ret = 0;
 cleanup:
    VIR_FREE(addresses);
    VIR_FREE(maxFuncsStr);
    return ret;
}


virNodeDeviceCapNetDefFormat(virBufferPtr buf,
                             const virNodeDevCapData *data)
{
    size_t i;

    virBufferEscapeString(buf, "<interface>%s</interface>\n",
                          data->net.ifname);
    if (data->net.address)
        virBufferEscapeString(buf, "<address>%s</address>\n",
                              data->net.address);
    virInterfaceLinkFormat(buf, &data->net.lnk);
    if (data->net.features) {
        for (i = 0; i < VIR_NET_DEV_FEAT_LAST; i++) {
            if (virBitmapIsBitSet(data->net.features, i)) {
                virBufferAsprintf(buf, "<feature name='%s'/>\n",
                                  virNetDevFeatureTypeToString(i));
            }
        }
    }
    if (data->net.subtype != VIR_NODE_DEV_CAP_NET_LAST) {
        const char *subtyp =
            virNodeDevNetCapTypeToString(data->net.subtype);
        virBufferEscapeString(buf, "<capability type='%s'/>\n",
                              subtyp);
    }
}


virNodeDeviceDefFormat(const virNodeDeviceDef *def)
{
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
    virNodeDevCapsDefPtr caps;
    size_t i = 0;

    virBufferAddLit(&buf, "<device>\n");
    virBufferAdjustIndent(&buf, 2);
    virBufferEscapeString(&buf, "<name>%s</name>\n", def->name);
    virBufferEscapeString(&buf, "<path>%s</path>\n", def->sysfs_path);
    if (def->devnode)
        virBufferEscapeString(&buf, "<devnode type='dev'>%s</devnode>\n",
                              def->devnode);
    if (def->devlinks) {
        for (i = 0; def->devlinks[i]; i++)
            virBufferEscapeString(&buf, "<devnode type='link'>%s</devnode>\n",
                                  def->devlinks[i]);
    }
    if (def->parent)
        virBufferEscapeString(&buf, "<parent>%s</parent>\n", def->parent);
    if (def->driver) {
        virBufferAddLit(&buf, "<driver>\n");
        virBufferAdjustIndent(&buf, 2);
        virBufferEscapeString(&buf, "<name>%s</name>\n", def->driver);
        virBufferAdjustIndent(&buf, -2);
        virBufferAddLit(&buf, "</driver>\n");
    }

    for (caps = def->caps; caps; caps = caps->next) {
        virNodeDevCapDataPtr data = &caps->data;

        virBufferAsprintf(&buf, "<capability type='%s'>\n",
                          virNodeDevCapTypeToString(caps->data.type));
        virBufferAdjustIndent(&buf, 2);
        switch (caps->data.type) {
        case VIR_NODE_DEV_CAP_SYSTEM:
            virNodeDeviceCapSystemDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_PCI_DEV:
            virNodeDeviceCapPCIDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_USB_DEV:
            virNodeDeviceCapUSBDevDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_USB_INTERFACE:
            virNodeDeviceCapUSBInterfaceDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_NET:
            virNodeDeviceCapNetDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_SCSI_HOST:
            virNodeDeviceCapSCSIHostDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_SCSI_TARGET:
            virBufferEscapeString(&buf, "<target>%s</target>\n",
                                  data->scsi_target.name);
            if (data->scsi_target.flags & VIR_NODE_DEV_CAP_FLAG_FC_RPORT) {
                virBufferAddLit(&buf, "<capability type='fc_remote_port'>\n");
                virBufferAdjustIndent(&buf, 2);
                virBufferAsprintf(&buf, "<rport>%s</rport>\n",
                                  data->scsi_target.rport);
                virBufferAsprintf(&buf, "<wwpn>%s</wwpn>\n",
                                  data->scsi_target.wwpn);
                virBufferAdjustIndent(&buf, -2);
                virBufferAddLit(&buf, "</capability>\n");
            }
            break;
        case VIR_NODE_DEV_CAP_SCSI:
            virNodeDeviceCapSCSIDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_STORAGE:
            virNodeDeviceCapStorageDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_SCSI_GENERIC:
            virBufferEscapeString(&buf, "<char>%s</char>\n",
                                  data->sg.path);
            break;
        case VIR_NODE_DEV_CAP_DRM:
            virBufferEscapeString(&buf, "<type>%s</type>\n", virNodeDevDRMTypeToString(data->drm.type));
            break;
        case VIR_NODE_DEV_CAP_MDEV:
            virNodeDeviceCapMdevDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_CCW_DEV:
        case VIR_NODE_DEV_CAP_CSS_DEV:
            virNodeDeviceCapCCWDefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_VDPA:
            virNodeDeviceCapVDPADefFormat(&buf, data);
            break;
        case VIR_NODE_DEV_CAP_MDEV_TYPES:
        case VIR_NODE_DEV_CAP_FC_HOST:
        case VIR_NODE_DEV_CAP_VPORTS:
        case VIR_NODE_DEV_CAP_LAST:
            break;
        }

        virBufferAdjustIndent(&buf, -2);
        virBufferAddLit(&buf, "</capability>\n");
    }

    virBufferAdjustIndent(&buf, -2);
    virBufferAddLit(&buf, "</device>\n");

    return virBufferContentAndReset(&buf);
}


virNodeDevCapsDefParseIntOptional(const char *xpath,
                                  xmlXPathContextPtr ctxt,
                                  int *value,
                                  virNodeDeviceDefPtr def,
                                  const char *invalid_error_fmt)
{
    int ret;
    int val;

    ret = virXPathInt(xpath, ctxt, &val);
    if (ret < -1) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       invalid_error_fmt,
                       def->name);
        return -1;
    } else if (ret == -1) {
        return 0;
    }
    *value = val;
    return 1;
}


virNodeDeviceGetSCSITargetCaps(const char *sysfsPath,
                               virNodeDevCapSCSITargetPtr scsi_target)
{
    int ret = -1;
    char *dir = NULL, *rport = NULL;

    VIR_DEBUG("Checking if '%s' is an FC remote port", scsi_target->name);

    /* /sys/devices/[...]/host0/rport-0:0-0/target0:0:0 -> rport-0:0-0 */
    dir = g_path_get_dirname(sysfsPath);

    rport = g_path_get_basename(dir);

    if (!virFCIsCapableRport(rport))
        goto cleanup;

    VIR_FREE(scsi_target->rport);
    scsi_target->rport = g_steal_pointer(&rport);

    if (virFCReadRportValue(scsi_target->rport, "port_name",
                            &scsi_target->wwpn) < 0) {
        VIR_WARN("Failed to read port_name for '%s'", scsi_target->rport);
        goto cleanup;
    }

    scsi_target->flags |= VIR_NODE_DEV_CAP_FLAG_FC_RPORT;
    ret = 0;

 cleanup:
    if (ret < 0) {
        VIR_FREE(scsi_target->rport);
        VIR_FREE(scsi_target->wwpn);
        scsi_target->flags &= ~VIR_NODE_DEV_CAP_FLAG_FC_RPORT;
    }
    VIR_FREE(rport);
    VIR_FREE(dir);

    return ret;
}