ctnetlink_glue_parse_ct(const struct nlattr *cda[], struct nf_conn *ct)
{
	int err;

	if (cda[CTA_TIMEOUT]) {
		err = ctnetlink_change_timeout(ct, cda);
		if (err < 0)
			return err;
	}
	if (cda[CTA_STATUS]) {
		err = ctnetlink_update_status(ct, cda);
		if (err < 0)
			return err;
	}
	if (cda[CTA_HELP]) {
		err = ctnetlink_change_helper(ct, cda);
		if (err < 0)
			return err;
	}
	if (cda[CTA_LABELS]) {
		err = ctnetlink_attach_labels(ct, cda);
		if (err < 0)
			return err;
	}
#if defined(CONFIG_NF_CONNTRACK_MARK)
	if (cda[CTA_MARK]) {
		ctnetlink_change_mark(ct, cda);
	}
#endif
	return 0;
}


static int ctnetlink_dump_tuples_proto(struct sk_buff *skb,
				const struct nf_conntrack_tuple *tuple,
				const struct nf_conntrack_l4proto *l4proto)
{
	int ret = 0;
	struct nlattr *nest_parms;

	nest_parms = nla_nest_start(skb, CTA_TUPLE_PROTO);
	if (!nest_parms)
		goto nla_put_failure;
	if (nla_put_u8(skb, CTA_PROTO_NUM, tuple->dst.protonum))
		goto nla_put_failure;

	if (likely(l4proto->tuple_to_nlattr))
		ret = l4proto->tuple_to_nlattr(skb, tuple);

	nla_nest_end(skb, nest_parms);

	return ret;

nla_put_failure:
	return -1;
}


static int ctnetlink_del_conntrack(struct net *net, struct sock *ctnl,
				   struct sk_buff *skb,
				   const struct nlmsghdr *nlh,
				   const struct nlattr * const cda[],
				   struct netlink_ext_ack *extack)
{
	struct nf_conntrack_tuple_hash *h;
	struct nf_conntrack_tuple tuple;
	struct nf_conn *ct;
	struct nfgenmsg *nfmsg = nlmsg_data(nlh);
	struct nf_conntrack_zone zone;
	int err;

	err = ctnetlink_parse_zone(cda[CTA_ZONE], &zone);
	if (err < 0)
		return err;

	if (cda[CTA_TUPLE_ORIG])
		err = ctnetlink_parse_tuple(cda, &tuple, CTA_TUPLE_ORIG,
					    nfmsg->nfgen_family, &zone);
	else if (cda[CTA_TUPLE_REPLY])
		err = ctnetlink_parse_tuple(cda, &tuple, CTA_TUPLE_REPLY,
					    nfmsg->nfgen_family, &zone);
	else {
		u_int8_t u3 = nfmsg->version ? nfmsg->nfgen_family : AF_UNSPEC;

		return ctnetlink_flush_conntrack(net, cda,
						 NETLINK_CB(skb).portid,
						 nlmsg_report(nlh), u3);
	}

	if (err < 0)
		return err;

	h = nf_conntrack_find_get(net, &zone, &tuple);
	if (!h)
		return -ENOENT;

	ct = nf_ct_tuplehash_to_ctrack(h);

	if (test_bit(IPS_OFFLOAD_BIT, &ct->status)) {
		nf_ct_put(ct);
		return -EBUSY;
	}

	if (cda[CTA_ID]) {
		__be32 id = nla_get_be32(cda[CTA_ID]);

		if (id != (__force __be32)nf_ct_get_id(ct)) {
			nf_ct_put(ct);
			return -ENOENT;
		}
	}

	nf_ct_delete(ct, NETLINK_CB(skb).portid, nlmsg_report(nlh));
	nf_ct_put(ct);

	return 0;
}


static int ctnetlink_dump_id(struct sk_buff *skb, const struct nf_conn *ct)
{
	__be32 id = (__force __be32)nf_ct_get_id(ct);

	if (nla_put_be32(skb, CTA_ID, id))
		goto nla_put_failure;
	return 0;

nla_put_failure:
	return -1;
}


static int ctnetlink_change_synproxy(struct nf_conn *ct,
				     const struct nlattr * const cda[])
{
	struct nf_conn_synproxy *synproxy = nfct_synproxy(ct);
	struct nlattr *tb[CTA_SYNPROXY_MAX + 1];
	int err;

	if (!synproxy)
		return 0;

	err = nla_parse_nested_deprecated(tb, CTA_SYNPROXY_MAX,
					  cda[CTA_SYNPROXY], synproxy_policy,
					  NULL);
	if (err < 0)
		return err;

	if (!tb[CTA_SYNPROXY_ISN] ||
	    !tb[CTA_SYNPROXY_ITS] ||
	    !tb[CTA_SYNPROXY_TSOFF])
		return -EINVAL;

	synproxy->isn = ntohl(nla_get_be32(tb[CTA_SYNPROXY_ISN]));
	synproxy->its = ntohl(nla_get_be32(tb[CTA_SYNPROXY_ITS]));
	synproxy->tsoff = ntohl(nla_get_be32(tb[CTA_SYNPROXY_TSOFF]));

	return 0;
}


ctnetlink_dump_acct(struct sk_buff *skb, const struct nf_conn *ct, int type)
{
	struct nf_conn_acct *acct = nf_conn_acct_find(ct);

	if (!acct)
		return 0;

	if (dump_counters(skb, acct, IP_CT_DIR_ORIGINAL, type) < 0)
		return -1;
	if (dump_counters(skb, acct, IP_CT_DIR_REPLY, type) < 0)
		return -1;

	return 0;
}


static inline int ctnetlink_secctx_size(const struct nf_conn *ct)
{
#ifdef CONFIG_NF_CONNTRACK_SECMARK
	int len, ret;

	ret = security_secid_to_secctx(ct->secmark, NULL, &len);
	if (ret)
		return 0;

	return nla_total_size(0) /* CTA_SECCTX */
	       + nla_total_size(sizeof(char) * len); /* CTA_SECCTX_NAME */
#else
	return 0;
#endif
}


static int ctnetlink_dump_extinfo(struct sk_buff *skb,
				  struct nf_conn *ct, u32 type)
{
	if (ctnetlink_dump_acct(skb, ct, type) < 0 ||
	    ctnetlink_dump_timestamp(skb, ct) < 0 ||
	    ctnetlink_dump_helpinfo(skb, ct) < 0 ||
	    ctnetlink_dump_labels(skb, ct) < 0 ||
	    ctnetlink_dump_ct_seq_adj(skb, ct) < 0 ||
	    ctnetlink_dump_ct_synproxy(skb, ct) < 0)
		return -1;

	return 0;
}


static int decode_attr_space_total(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res)
{
	__be32 *p;
	int status = 0;

	*res = 0;
	if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_TOTAL - 1U)))
		return -EIO;
	if (likely(bitmap[1] & FATTR4_WORD1_SPACE_TOTAL)) {
		p = xdr_inline_decode(xdr, 8);
		if (unlikely(!p))
			return -EIO;
		xdr_decode_hyper(p, res);
		bitmap[1] &= ~FATTR4_WORD1_SPACE_TOTAL;
	}
	dprintk("%s: space total=%Lu\n", __func__, (unsigned long long)*res);
	return status;
}


static inline void encode_claim_fh(struct xdr_stream *xdr)
{
	__be32 *p;

	p = reserve_space(xdr, 4);
	*p = cpu_to_be32(NFS4_OPEN_CLAIM_FH);
}


static void nfs_finish_clear_delegation_stateid(struct nfs4_state *state,
		const nfs4_stateid *stateid)
{
	nfs_remove_bad_delegation(state->inode, stateid);
	nfs_state_clear_delegation(state);
}


static int _nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
		u64 cookie, struct page **pages, unsigned int count, bool plus)
{
	struct inode		*dir = d_inode(dentry);
	struct nfs4_readdir_arg args = {
		.fh = NFS_FH(dir),
		.pages = pages,
		.pgbase = 0,
		.count = count,
		.bitmask = NFS_SERVER(d_inode(dentry))->attr_bitmask,
		.plus = plus,
	};
	struct nfs4_readdir_res res;
	struct rpc_message msg = {
		.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READDIR],
		.rpc_argp = &args,
		.rpc_resp = &res,
		.rpc_cred = cred,
	};
	int			status;

	dprintk("%s: dentry = %pd2, cookie = %Lu\n", __func__,
			dentry,
			(unsigned long long)cookie);
	nfs4_setup_readdir(cookie, NFS_I(dir)->cookieverf, dentry, &args);
	res.pgbase = args.pgbase;
	status = nfs4_call_sync(NFS_SERVER(dir)->client, NFS_SERVER(dir), &msg, &args.seq_args, &res.seq_res, 0);
	if (status >= 0) {
		memcpy(NFS_I(dir)->cookieverf, res.verifier.data, NFS4_VERIFIER_SIZE);
		status += args.pgbase;
	}

	nfs_invalidate_atime(dir);

	dprintk("%s: returns %d\n", __func__, status);
	return status;
}


static void nfs41_release_slot(struct nfs4_slot *slot)
{
	struct nfs4_session *session;
	struct nfs4_slot_table *tbl;
	bool send_new_highest_used_slotid = false;

	if (!slot)
		return;
	tbl = slot->table;
	session = tbl->session;

	/* Bump the slot sequence number */
	if (slot->seq_done)
		slot->seq_nr++;
	slot->seq_done = 0;

	spin_lock(&tbl->slot_tbl_lock);
	/* Be nice to the server: try to ensure that the last transmitted
	 * value for highest_user_slotid <= target_highest_slotid
	 */
	if (tbl->highest_used_slotid > tbl->target_highest_slotid)
		send_new_highest_used_slotid = true;

	if (nfs41_wake_and_assign_slot(tbl, slot)) {
		send_new_highest_used_slotid = false;
		goto out_unlock;
	}
	nfs4_free_slot(tbl, slot);

	if (tbl->highest_used_slotid != NFS4_NO_SLOT)
		send_new_highest_used_slotid = false;
out_unlock:
	spin_unlock(&tbl->slot_tbl_lock);
	if (send_new_highest_used_slotid)
		nfs41_notify_server(session->clp);
	if (waitqueue_active(&tbl->slot_waitq))
		wake_up_all(&tbl->slot_waitq);
}


static int nfs40_sequence_done(struct rpc_task *task,
			       struct nfs4_sequence_res *res)
{
	if (res->sr_slot != NULL)
		nfs40_sequence_free_slot(res);
	return 1;
}


static void encode_fsinfo(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)
{
	encode_getattr(xdr, nfs4_fsinfo_bitmap, bitmask,
			ARRAY_SIZE(nfs4_fsinfo_bitmap), hdr);
}


static void nfs40_clear_delegation_stateid(struct nfs4_state *state)
{
	if (rcu_access_pointer(NFS_I(state->inode)->delegation) != NULL)
		nfs_finish_clear_delegation_stateid(state, NULL);
}


static int decode_test_stateid(struct xdr_stream *xdr,
			       struct nfs41_test_stateid_res *res)
{
	__be32 *p;
	int status;
	int num_res;

	status = decode_op_hdr(xdr, OP_TEST_STATEID);
	if (status)
		return status;

	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	num_res = be32_to_cpup(p++);
	if (num_res != 1)
		return -EIO;

	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	res->status = be32_to_cpup(p++);

	return status;
}


static int decode_pathname(struct xdr_stream *xdr, struct nfs4_pathname *path)
{
	u32 n;
	__be32 *p;
	int status = 0;

	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	n = be32_to_cpup(p);
	if (n == 0)
		goto root_path;
	dprintk("pathname4: ");
	if (n > NFS4_PATHNAME_MAXCOMPONENTS) {
		dprintk("cannot parse %d components in path\n", n);
		goto out_eio;
	}
	for (path->ncomponents = 0; path->ncomponents < n; path->ncomponents++) {
		struct nfs4_string *component = &path->components[path->ncomponents];
		status = decode_opaque_inline(xdr, &component->len, &component->data);
		if (unlikely(status != 0))
			goto out_eio;
		ifdebug (XDR)
			pr_cont("%s%.*s ",
				(path->ncomponents != n ? "/ " : ""),
				component->len, component->data);
	}
out:
	return status;
root_path:
/* a root pathname is sent as a zero component4 */
	path->ncomponents = 1;
	path->components[0].len=0;
	path->components[0].data=NULL;
	dprintk("pathname4: /\n");
	goto out;
out_eio:
	dprintk(" status %d", status);
	status = -EIO;
	goto out;
}


static void nfs4_release_lockowner_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs_release_lockowner_data *data = calldata;
	struct nfs_server *server = data->server;
	nfs4_setup_sequence(server->nfs_client, &data->args.seq_args,
			   &data->res.seq_res, task);
	data->args.lock_owner.clientid = server->nfs_client->cl_clientid;
	data->timestamp = jiffies;
}


static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, const nfs4_stateid *stateid, int issync)
{
	struct nfs4_delegreturndata *data;
	struct nfs_server *server = NFS_SERVER(inode);
	struct rpc_task *task;
	struct rpc_message msg = {
		.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DELEGRETURN],
		.rpc_cred = cred,
	};
	struct rpc_task_setup task_setup_data = {
		.rpc_client = server->client,
		.rpc_message = &msg,
		.callback_ops = &nfs4_delegreturn_ops,
		.flags = RPC_TASK_ASYNC | RPC_TASK_TIMEOUT,
	};
	int status = 0;

	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (data == NULL)
		return -ENOMEM;
	nfs4_init_sequence(&data->args.seq_args, &data->res.seq_res, 1, 0);

	nfs4_state_protect(server->nfs_client,
			NFS_SP4_MACH_CRED_CLEANUP,
			&task_setup_data.rpc_client, &msg);

	data->args.fhandle = &data->fh;
	data->args.stateid = &data->stateid;
	data->args.bitmask = server->cache_consistency_bitmask;
	nfs_copy_fh(&data->fh, NFS_FH(inode));
	nfs4_stateid_copy(&data->stateid, stateid);
	data->res.fattr = &data->fattr;
	data->res.server = server;
	data->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
	data->lr.arg.ld_private = &data->lr.ld_private;
	nfs_fattr_init(data->res.fattr);
	data->timestamp = jiffies;
	data->rpc_status = 0;
	data->lr.roc = pnfs_roc(inode, &data->lr.arg, &data->lr.res, cred);
	data->inode = nfs_igrab_and_active(inode);
	if (data->inode) {
		if (data->lr.roc) {
			data->args.lr_args = &data->lr.arg;
			data->res.lr_res = &data->lr.res;
		}
	} else if (data->lr.roc) {
		pnfs_roc_release(&data->lr.arg, &data->lr.res, 0);
		data->lr.roc = false;
	}

	task_setup_data.callback_data = data;
	msg.rpc_argp = &data->args;
	msg.rpc_resp = &data->res;
	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (!issync)
		goto out;
	status = rpc_wait_for_completion_task(task);
	if (status != 0)
		goto out;
	status = data->rpc_status;
out:
	rpc_put_task(task);
	return status;
}


static int decode_attr_group(struct xdr_stream *xdr, uint32_t *bitmap,
		const struct nfs_server *server, kgid_t *gid,
		struct nfs4_string *group_name)
{
	ssize_t len;
	char *p;

	*gid = make_kgid(&init_user_ns, -2);
	if (unlikely(bitmap[1] & (FATTR4_WORD1_OWNER_GROUP - 1U)))
		return -EIO;
	if (!(bitmap[1] & FATTR4_WORD1_OWNER_GROUP))
		return 0;
	bitmap[1] &= ~FATTR4_WORD1_OWNER_GROUP;

	if (group_name != NULL) {
		len = decode_nfs4_string(xdr, group_name, GFP_NOIO);
		if (len <= 0)
			goto out;
		dprintk("%s: name=%s\n", __func__, group_name->data);
		return NFS_ATTR_FATTR_GROUP_NAME;
	} else {
		len = xdr_stream_decode_opaque_inline(xdr, (void **)&p,
				XDR_MAX_NETOBJ);
		if (len <= 0 || nfs_map_group_to_gid(server, p, len, gid) != 0)
			goto out;
		dprintk("%s: gid=%d\n", __func__, (int)from_kgid(&init_user_ns, *gid));
		return NFS_ATTR_FATTR_GROUP;
	}
out:
	if (len == -EBADMSG)
		return -EIO;
	return 0;
}


static void __nfs4_free_revoked_stateid(struct nfs_server *server,
		nfs4_stateid *stateid,
		const struct cred *cred)
{
	stateid->type = NFS4_REVOKED_STATEID_TYPE;
	nfs4_test_and_free_stateid(server, stateid, cred);
}


static int decode_write(struct xdr_stream *xdr, struct nfs_pgio_res *res)
{
	__be32 *p;
	int status;

	status = decode_op_hdr(xdr, OP_WRITE);
	if (status)
		return status;

	p = xdr_inline_decode(xdr, 8);
	if (unlikely(!p))
		return -EIO;
	res->count = be32_to_cpup(p++);
	res->verf->committed = be32_to_cpup(p++);
	return decode_write_verifier(xdr, &res->verf->verifier);
}


static int nfs4_handle_delegation_recall_error(struct nfs_server *server, struct nfs4_state *state, const nfs4_stateid *stateid, struct file_lock *fl, int err)
{
	switch (err) {
		default:
			printk(KERN_ERR "NFS: %s: unhandled error "
					"%d.\n", __func__, err);
		case 0:
		case -ENOENT:
		case -EAGAIN:
		case -ESTALE:
		case -ETIMEDOUT:
			break;
		case -NFS4ERR_BADSESSION:
		case -NFS4ERR_BADSLOT:
		case -NFS4ERR_BAD_HIGH_SLOT:
		case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
		case -NFS4ERR_DEADSESSION:
			return -EAGAIN;
		case -NFS4ERR_STALE_CLIENTID:
		case -NFS4ERR_STALE_STATEID:
			/* Don't recall a delegation if it was lost */
			nfs4_schedule_lease_recovery(server->nfs_client);
			return -EAGAIN;
		case -NFS4ERR_MOVED:
			nfs4_schedule_migration_recovery(server);
			return -EAGAIN;
		case -NFS4ERR_LEASE_MOVED:
			nfs4_schedule_lease_moved_recovery(server->nfs_client);
			return -EAGAIN;
		case -NFS4ERR_DELEG_REVOKED:
		case -NFS4ERR_ADMIN_REVOKED:
		case -NFS4ERR_EXPIRED:
		case -NFS4ERR_BAD_STATEID:
		case -NFS4ERR_OPENMODE:
			nfs_inode_find_state_and_recover(state->inode,
					stateid);
			nfs4_schedule_stateid_recovery(server, state);
			return -EAGAIN;
		case -NFS4ERR_DELAY:
		case -NFS4ERR_GRACE:
			ssleep(1);
			return -EAGAIN;
		case -ENOMEM:
		case -NFS4ERR_DENIED:
			if (fl) {
				struct nfs4_lock_state *lsp = fl->fl_u.nfs4_fl.owner;
				if (lsp)
					set_bit(NFS_LOCK_LOST, &lsp->ls_flags);
			}
			return 0;
	}
	return err;
}


int nfs4_open_delegation_recall(struct nfs_open_context *ctx,
		struct nfs4_state *state, const nfs4_stateid *stateid)
{
	struct nfs_server *server = NFS_SERVER(state->inode);
	struct nfs4_opendata *opendata;
	int err = 0;

	opendata = nfs4_open_recoverdata_alloc(ctx, state,
			NFS4_OPEN_CLAIM_DELEG_CUR_FH);
	if (IS_ERR(opendata))
		return PTR_ERR(opendata);
	nfs4_stateid_copy(&opendata->o_arg.u.delegation, stateid);
	if (!test_bit(NFS_O_RDWR_STATE, &state->flags)) {
		err = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE);
		if (err)
			goto out;
	}
	if (!test_bit(NFS_O_WRONLY_STATE, &state->flags)) {
		err = nfs4_open_recover_helper(opendata, FMODE_WRITE);
		if (err)
			goto out;
	}
	if (!test_bit(NFS_O_RDONLY_STATE, &state->flags)) {
		err = nfs4_open_recover_helper(opendata, FMODE_READ);
		if (err)
			goto out;
	}
	nfs_state_clear_delegation(state);
out:
	nfs4_opendata_put(opendata);
	return nfs4_handle_delegation_recall_error(server, state, stateid, NULL, err);
}


static int nfs4_xattr_get_nfs4_acl(const struct xattr_handler *handler,
				   struct dentry *unused, struct inode *inode,
				   const char *key, void *buf, size_t buflen)
{
	return nfs4_proc_get_acl(inode, buf, buflen);
}


static void nfs4_sync_open_stateid(nfs4_stateid *dst,
		struct nfs4_state *state)
{
	__be32 seqid_open;
	u32 dst_seqid;
	int seq;

	for (;;) {
		if (!nfs4_valid_open_stateid(state))
			break;
		seq = read_seqbegin(&state->seqlock);
		if (!nfs4_state_match_open_stateid_other(state, dst)) {
			nfs4_stateid_copy(dst, &state->open_stateid);
			if (read_seqretry(&state->seqlock, seq))
				continue;
			break;
		}
		seqid_open = state->open_stateid.seqid;
		if (read_seqretry(&state->seqlock, seq))
			continue;

		dst_seqid = be32_to_cpu(dst->seqid);
		if ((s32)(dst_seqid - be32_to_cpu(seqid_open)) < 0)
			dst->seqid = seqid_open;
		break;
	}
}


nfs4_label_release_security(struct nfs4_label *label)
{
	if (label)
		security_release_secctx(label->label, label->len);
}


static int nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock *request)
{
	struct nfs4_exception exception = {
		.interruptible = true,
	};
	int err;

	do {
		err = _nfs4_proc_getlk(state, cmd, request);
		trace_nfs4_get_lock(request, state, cmd, err);
		err = nfs4_handle_exception(NFS_SERVER(state->inode), err,
				&exception);
	} while (exception.retry);
	return err;
}


static int _nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock *request)
{
	struct inode *inode = state->inode;
	struct nfs_server *server = NFS_SERVER(inode);
	struct nfs_client *clp = server->nfs_client;
	struct nfs_lockt_args arg = {
		.fh = NFS_FH(inode),
		.fl = request,
	};
	struct nfs_lockt_res res = {
		.denied = request,
	};
	struct rpc_message msg = {
		.rpc_proc	= &nfs4_procedures[NFSPROC4_CLNT_LOCKT],
		.rpc_argp	= &arg,
		.rpc_resp	= &res,
		.rpc_cred	= state->owner->so_cred,
	};
	struct nfs4_lock_state *lsp;
	int status;

	arg.lock_owner.clientid = clp->cl_clientid;
	status = nfs4_set_lock_state(state, request);
	if (status != 0)
		goto out;
	lsp = request->fl_u.nfs4_fl.owner;
	arg.lock_owner.id = lsp->ls_seqid.owner_id;
	arg.lock_owner.s_dev = server->s_dev;
	status = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1);
	switch (status) {
		case 0:
			request->fl_type = F_UNLCK;
			break;
		case -NFS4ERR_DENIED:
			status = 0;
	}
	request->fl_ops->fl_release_private(request);
	request->fl_ops = NULL;
out:
	return status;
}


static int decode_attr_space_free(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res)
{
	__be32 *p;
	int status = 0;

	*res = 0;
	if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_FREE - 1U)))
		return -EIO;
	if (likely(bitmap[1] & FATTR4_WORD1_SPACE_FREE)) {
		p = xdr_inline_decode(xdr, 8);
		if (unlikely(!p))
			return -EIO;
		xdr_decode_hyper(p, res);
		bitmap[1] &= ~FATTR4_WORD1_SPACE_FREE;
	}
	dprintk("%s: space free=%Lu\n", __func__, (unsigned long long)*res);
	return status;
}


static void nfs4_xdr_enc_lookupp(struct rpc_rqst *req, struct xdr_stream *xdr,
		const void *data)
{
	const struct nfs4_lookupp_arg *args = data;
	struct compound_hdr hdr = {
		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
	};

	encode_compound_hdr(xdr, req, &hdr);
	encode_sequence(xdr, &args->seq_args, &hdr);
	encode_putfh(xdr, args->fh, &hdr);
	encode_lookupp(xdr, &hdr);
	encode_getfh(xdr, &hdr);
	encode_getfattr(xdr, args->bitmask, &hdr);
	encode_nops(&hdr);
}


static void nfs4_xdr_enc_open_downgrade(struct rpc_rqst *req,
					struct xdr_stream *xdr,
					const void *data)
{
	const struct nfs_closeargs *args = data;
	struct compound_hdr hdr = {
		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
	};

	encode_compound_hdr(xdr, req, &hdr);
	encode_sequence(xdr, &args->seq_args, &hdr);
	encode_putfh(xdr, args->fh, &hdr);
	if (args->lr_args)
		encode_layoutreturn(xdr, args->lr_args, &hdr);
	encode_open_downgrade(xdr, args, &hdr);
	encode_nops(&hdr);
}


nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle,
			   struct nfs_fsinfo *info, struct nfs4_secinfo_flavors *flavors)
{
	struct nfs4_exception exception = {
		.interruptible = true,
	};
	int err;
	do {
		/* first try using integrity protection */
		err = -NFS4ERR_WRONGSEC;

		/* try to use integrity protection with machine cred */
		if (_nfs4_is_integrity_protected(server->nfs_client))
			err = _nfs41_proc_secinfo_no_name(server, fhandle, info,
							  flavors, true);

		/*
		 * if unable to use integrity protection, or SECINFO with
		 * integrity protection returns NFS4ERR_WRONGSEC (which is
		 * disallowed by spec, but exists in deployed servers) use
		 * the current filesystem's rpc_client and the user cred.
		 */
		if (err == -NFS4ERR_WRONGSEC)
			err = _nfs41_proc_secinfo_no_name(server, fhandle, info,
							  flavors, false);

		switch (err) {
		case 0:
		case -NFS4ERR_WRONGSEC:
		case -ENOTSUPP:
			goto out;
		default:
			err = nfs4_handle_exception(server, err, &exception);
		}
	} while (exception.retry);
out:
	return err;
}


static int decode_open_downgrade(struct xdr_stream *xdr, struct nfs_closeres *res)
{
	int status;

	status = decode_op_hdr(xdr, OP_OPEN_DOWNGRADE);
	if (status != -EIO)
		nfs_increment_open_seqid(status, res->seqid);
	if (!status)
		status = decode_open_stateid(xdr, &res->stateid);
	return status;
}


nfs4_listxattr_nfs4_label(struct inode *inode, char *list, size_t list_len)
{
	return 0;
}


int nfs4_proc_commit(struct file *dst, __u64 offset, __u32 count, struct nfs_commitres *res)
{
	struct nfs_commitargs args = {
		.offset = offset,
		.count = count,
	};
	struct nfs_server *dst_server = NFS_SERVER(file_inode(dst));
	struct nfs4_exception exception = { };
	int status;

	do {
		status = _nfs4_proc_commit(dst, &args, res);
		status = nfs4_handle_exception(dst_server, status, &exception);
	} while (exception.retry);

	return status;
}


int nfs4_destroy_clientid(struct nfs_client *clp)
{
	const struct cred *cred;
	int ret = 0;

	if (clp->cl_mvops->minor_version < 1)
		goto out;
	if (clp->cl_exchange_flags == 0)
		goto out;
	if (clp->cl_preserve_clid)
		goto out;
	cred = nfs4_get_clid_cred(clp);
	ret = nfs4_proc_destroy_clientid(clp, cred);
	put_cred(cred);
	switch (ret) {
	case 0:
	case -NFS4ERR_STALE_CLIENTID:
		clp->cl_exchange_flags = 0;
	}
out:
	return ret;
}


static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle)
{
	u32 bitmask[3] = {}, minorversion = server->nfs_client->cl_minorversion;
	struct nfs4_server_caps_arg args = {
		.fhandle = fhandle,
		.bitmask = bitmask,
	};
	struct nfs4_server_caps_res res = {};
	struct rpc_message msg = {
		.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SERVER_CAPS],
		.rpc_argp = &args,
		.rpc_resp = &res,
	};
	int status;
	int i;

	bitmask[0] = FATTR4_WORD0_SUPPORTED_ATTRS |
		     FATTR4_WORD0_FH_EXPIRE_TYPE |
		     FATTR4_WORD0_LINK_SUPPORT |
		     FATTR4_WORD0_SYMLINK_SUPPORT |
		     FATTR4_WORD0_ACLSUPPORT;
	if (minorversion)
		bitmask[2] = FATTR4_WORD2_SUPPATTR_EXCLCREAT;

	status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0);
	if (status == 0) {
		/* Sanity check the server answers */
		switch (minorversion) {
		case 0:
			res.attr_bitmask[1] &= FATTR4_WORD1_NFS40_MASK;
			res.attr_bitmask[2] = 0;
			break;
		case 1:
			res.attr_bitmask[2] &= FATTR4_WORD2_NFS41_MASK;
			break;
		case 2:
			res.attr_bitmask[2] &= FATTR4_WORD2_NFS42_MASK;
		}
		memcpy(server->attr_bitmask, res.attr_bitmask, sizeof(server->attr_bitmask));
		server->caps &= ~(NFS_CAP_ACLS|NFS_CAP_HARDLINKS|
				NFS_CAP_SYMLINKS|NFS_CAP_FILEID|
				NFS_CAP_MODE|NFS_CAP_NLINK|NFS_CAP_OWNER|
				NFS_CAP_OWNER_GROUP|NFS_CAP_ATIME|
				NFS_CAP_CTIME|NFS_CAP_MTIME|
				NFS_CAP_SECURITY_LABEL);
		if (res.attr_bitmask[0] & FATTR4_WORD0_ACL &&
				res.acl_bitmask & ACL4_SUPPORT_ALLOW_ACL)
			server->caps |= NFS_CAP_ACLS;
		if (res.has_links != 0)
			server->caps |= NFS_CAP_HARDLINKS;
		if (res.has_symlinks != 0)
			server->caps |= NFS_CAP_SYMLINKS;
		if (res.attr_bitmask[0] & FATTR4_WORD0_FILEID)
			server->caps |= NFS_CAP_FILEID;
		if (res.attr_bitmask[1] & FATTR4_WORD1_MODE)
			server->caps |= NFS_CAP_MODE;
		if (res.attr_bitmask[1] & FATTR4_WORD1_NUMLINKS)
			server->caps |= NFS_CAP_NLINK;
		if (res.attr_bitmask[1] & FATTR4_WORD1_OWNER)
			server->caps |= NFS_CAP_OWNER;
		if (res.attr_bitmask[1] & FATTR4_WORD1_OWNER_GROUP)
			server->caps |= NFS_CAP_OWNER_GROUP;
		if (res.attr_bitmask[1] & FATTR4_WORD1_TIME_ACCESS)
			server->caps |= NFS_CAP_ATIME;
		if (res.attr_bitmask[1] & FATTR4_WORD1_TIME_METADATA)
			server->caps |= NFS_CAP_CTIME;
		if (res.attr_bitmask[1] & FATTR4_WORD1_TIME_MODIFY)
			server->caps |= NFS_CAP_MTIME;
#ifdef CONFIG_NFS_V4_SECURITY_LABEL
		if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL)
			server->caps |= NFS_CAP_SECURITY_LABEL;
#endif
		memcpy(server->attr_bitmask_nl, res.attr_bitmask,
				sizeof(server->attr_bitmask));
		server->attr_bitmask_nl[2] &= ~FATTR4_WORD2_SECURITY_LABEL;

		memcpy(server->cache_consistency_bitmask, res.attr_bitmask, sizeof(server->cache_consistency_bitmask));
		server->cache_consistency_bitmask[0] &= FATTR4_WORD0_CHANGE|FATTR4_WORD0_SIZE;
		server->cache_consistency_bitmask[1] &= FATTR4_WORD1_TIME_METADATA|FATTR4_WORD1_TIME_MODIFY;
		server->cache_consistency_bitmask[2] = 0;

		/* Avoid a regression due to buggy server */
		for (i = 0; i < ARRAY_SIZE(res.exclcreat_bitmask); i++)
			res.exclcreat_bitmask[i] &= res.attr_bitmask[i];
		memcpy(server->exclcreat_bitmask, res.exclcreat_bitmask,
			sizeof(server->exclcreat_bitmask));

		server->acl_bitmask = res.acl_bitmask;
		server->fh_expire_type = res.fh_expire_type;
	}

	return status;
}


nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync)
{
	struct rpc_message msg = {
		.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTCOMMIT],
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
		.rpc_cred = data->cred,
	};
	struct rpc_task_setup task_setup_data = {
		.task = &data->task,
		.rpc_client = NFS_CLIENT(data->args.inode),
		.rpc_message = &msg,
		.callback_ops = &nfs4_layoutcommit_ops,
		.callback_data = data,
	};
	struct rpc_task *task;
	int status = 0;

	dprintk("NFS: initiating layoutcommit call. sync %d "
		"lbw: %llu inode %lu\n", sync,
		data->args.lastbytewritten,
		data->args.inode->i_ino);

	if (!sync) {
		data->inode = nfs_igrab_and_active(data->args.inode);
		if (data->inode == NULL) {
			nfs4_layoutcommit_release(data);
			return -EAGAIN;
		}
		task_setup_data.flags = RPC_TASK_ASYNC;
	}
	nfs4_init_sequence(&data->args.seq_args, &data->res.seq_res, 1, 0);
	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (sync)
		status = task->tk_status;
	trace_nfs4_layoutcommit(data->args.inode, &data->args.stateid, status);
	dprintk("%s: status %d\n", __func__, status);
	rpc_put_task(task);
	return status;
}


static void nfs4_locku_prepare(struct rpc_task *task, void *data)
{
	struct nfs4_unlockdata *calldata = data;

	if (test_bit(NFS_CONTEXT_UNLOCK, &calldata->l_ctx->open_context->flags) &&
		nfs_async_iocounter_wait(task, calldata->l_ctx))
		return;

	if (nfs_wait_on_sequence(calldata->arg.seqid, task) != 0)
		goto out_wait;
	if (test_bit(NFS_LOCK_INITIALIZED, &calldata->lsp->ls_flags) == 0) {
		/* Note: exit _without_ running nfs4_locku_done */
		goto out_no_action;
	}
	calldata->timestamp = jiffies;
	if (nfs4_setup_sequence(calldata->server->nfs_client,
				&calldata->arg.seq_args,
				&calldata->res.seq_res,
				task) != 0)
		nfs_release_seqid(calldata->arg.seqid);
	return;
out_no_action:
	task->tk_action = NULL;
out_wait:
	nfs4_sequence_done(task, &calldata->res.seq_res);
}


static int decode_attr_nlink(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *nlink)
{
	__be32 *p;
	int ret = 0;

	*nlink = 1;
	if (unlikely(bitmap[1] & (FATTR4_WORD1_NUMLINKS - 1U)))
		return -EIO;
	if (likely(bitmap[1] & FATTR4_WORD1_NUMLINKS)) {
		p = xdr_inline_decode(xdr, 4);
		if (unlikely(!p))
			return -EIO;
		*nlink = be32_to_cpup(p);
		bitmap[1] &= ~FATTR4_WORD1_NUMLINKS;
		ret = NFS_ATTR_FATTR_NLINK;
	}
	dprintk("%s: nlink=%u\n", __func__, (unsigned int)*nlink);
	return ret;
}


static int decode_exchange_id(struct xdr_stream *xdr,
			      struct nfs41_exchange_id_res *res)
{
	__be32 *p;
	uint32_t dummy;
	char *dummy_str;
	int status;
	uint32_t impl_id_count;

	status = decode_op_hdr(xdr, OP_EXCHANGE_ID);
	if (status)
		return status;

	p = xdr_inline_decode(xdr, 8);
	if (unlikely(!p))
		return -EIO;
	xdr_decode_hyper(p, &res->clientid);
	p = xdr_inline_decode(xdr, 12);
	if (unlikely(!p))
		return -EIO;
	res->seqid = be32_to_cpup(p++);
	res->flags = be32_to_cpup(p++);

	res->state_protect.how = be32_to_cpup(p);
	switch (res->state_protect.how) {
	case SP4_NONE:
		break;
	case SP4_MACH_CRED:
		status = decode_op_map(xdr, &res->state_protect.enforce);
		if (status)
			return status;
		status = decode_op_map(xdr, &res->state_protect.allow);
		if (status)
			return status;
		break;
	default:
		WARN_ON_ONCE(1);
		return -EIO;
	}

	/* server_owner4.so_minor_id */
	p = xdr_inline_decode(xdr, 8);
	if (unlikely(!p))
		return -EIO;
	p = xdr_decode_hyper(p, &res->server_owner->minor_id);

	/* server_owner4.so_major_id */
	status = decode_opaque_inline(xdr, &dummy, &dummy_str);
	if (unlikely(status))
		return status;
	memcpy(res->server_owner->major_id, dummy_str, dummy);
	res->server_owner->major_id_sz = dummy;

	/* server_scope4 */
	status = decode_opaque_inline(xdr, &dummy, &dummy_str);
	if (unlikely(status))
		return status;
	memcpy(res->server_scope->server_scope, dummy_str, dummy);
	res->server_scope->server_scope_sz = dummy;

	/* Implementation Id */
	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	impl_id_count = be32_to_cpup(p++);

	if (impl_id_count) {
		/* nii_domain */
		status = decode_opaque_inline(xdr, &dummy, &dummy_str);
		if (unlikely(status))
			return status;
		memcpy(res->impl_id->domain, dummy_str, dummy);

		/* nii_name */
		status = decode_opaque_inline(xdr, &dummy, &dummy_str);
		if (unlikely(status))
			return status;
		memcpy(res->impl_id->name, dummy_str, dummy);

		/* nii_date */
		p = xdr_inline_decode(xdr, 12);
		if (unlikely(!p))
			return -EIO;
		p = xdr_decode_hyper(p, &res->impl_id->date.seconds);
		res->impl_id->date.nseconds = be32_to_cpup(p);

		/* if there's more than one entry, ignore the rest */
	}
	return 0;
}


static int nfs41_lock_expired(struct nfs4_state *state, struct file_lock *request)
{
	struct nfs4_lock_state *lsp;
	int status;

	status = nfs4_set_lock_state(state, request);
	if (status != 0)
		return status;
	lsp = request->fl_u.nfs4_fl.owner;
	if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) ||
	    test_bit(NFS_LOCK_LOST, &lsp->ls_flags))
		return 0;
	return nfs4_lock_expired(state, request);
}


static void nfs4_proc_rename_rpc_prepare(struct rpc_task *task, struct nfs_renamedata *data)
{
	nfs4_setup_sequence(NFS_SERVER(data->old_dir)->nfs_client,
			&data->args.seq_args,
			&data->res.seq_res,
			task);
}


static int nfs4_xdr_dec_free_stateid(struct rpc_rqst *rqstp,
				     struct xdr_stream *xdr,
				     void *data)
{
	struct nfs41_free_stateid_res *res = data;
	struct compound_hdr hdr;
	int status;

	status = decode_compound_hdr(xdr, &hdr);
	if (status)
		goto out;
	status = decode_sequence(xdr, &res->seq_res, rqstp);
	if (status)
		goto out;
	status = decode_free_stateid(xdr, res);
out:
	return status;
}


static int decode_attr_link_support(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res)
{
	__be32 *p;

	*res = 0;
	if (unlikely(bitmap[0] & (FATTR4_WORD0_LINK_SUPPORT - 1U)))
		return -EIO;
	if (likely(bitmap[0] & FATTR4_WORD0_LINK_SUPPORT)) {
		p = xdr_inline_decode(xdr, 4);
		if (unlikely(!p))
			return -EIO;
		*res = be32_to_cpup(p);
		bitmap[0] &= ~FATTR4_WORD0_LINK_SUPPORT;
	}
	dprintk("%s: link support=%s\n", __func__, *res == 0 ? "false" : "true");
	return 0;
}


static int _nfs41_proc_get_locations(struct inode *inode,
				     struct nfs4_fs_locations *locations,
				     struct page *page, const struct cred *cred)
{
	struct nfs_server *server = NFS_SERVER(inode);
	struct rpc_clnt *clnt = server->client;
	u32 bitmask[2] = {
		[0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS,
	};
	struct nfs4_fs_locations_arg args = {
		.fh		= NFS_FH(inode),
		.page		= page,
		.bitmask	= bitmask,
		.migration	= 1,		/* skip LOOKUP */
	};
	struct nfs4_fs_locations_res res = {
		.fs_locations	= locations,
		.migration	= 1,
	};
	struct rpc_message msg = {
		.rpc_proc	= &nfs4_procedures[NFSPROC4_CLNT_FS_LOCATIONS],
		.rpc_argp	= &args,
		.rpc_resp	= &res,
		.rpc_cred	= cred,
	};
	int status;

	nfs_fattr_init(&locations->fattr);
	locations->server = server;
	locations->nlocations = 0;

	nfs4_init_sequence(&args.seq_args, &res.seq_res, 0, 1);
	status = nfs4_call_sync_sequence(clnt, server, &msg,
					&args.seq_args, &res.seq_res);
	if (status == NFS4_OK &&
	    res.seq_res.sr_status_flags & SEQ4_STATUS_LEASE_MOVED)
		status = -NFS4ERR_LEASE_MOVED;
	return status;
}


static int nfs4_xdr_dec_rename(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
			       void *data)
{
	struct nfs_renameres *res = data;
	struct compound_hdr hdr;
	int status;

	status = decode_compound_hdr(xdr, &hdr);
	if (status)
		goto out;
	status = decode_sequence(xdr, &res->seq_res, rqstp);
	if (status)
		goto out;
	status = decode_putfh(xdr);
	if (status)
		goto out;
	status = decode_savefh(xdr);
	if (status)
		goto out;
	status = decode_putfh(xdr);
	if (status)
		goto out;
	status = decode_rename(xdr, &res->old_cinfo, &res->new_cinfo);
out:
	return status;
}


static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata,
		int flags, struct nfs_open_context *ctx)
{
	struct nfs4_state_owner *sp = opendata->owner;
	struct nfs_server *server = sp->so_server;
	struct dentry *dentry;
	struct nfs4_state *state;
	fmode_t acc_mode = _nfs4_ctx_to_accessmode(ctx);
	struct inode *dir = d_inode(opendata->dir);
	unsigned long dir_verifier;
	unsigned int seq;
	int ret;

	seq = raw_seqcount_begin(&sp->so_reclaim_seqcount);
	dir_verifier = nfs_save_change_attribute(dir);

	ret = _nfs4_proc_open(opendata, ctx);
	if (ret != 0)
		goto out;

	state = _nfs4_opendata_to_nfs4_state(opendata);
	ret = PTR_ERR(state);
	if (IS_ERR(state))
		goto out;
	ctx->state = state;
	if (server->caps & NFS_CAP_POSIX_LOCK)
		set_bit(NFS_STATE_POSIX_LOCKS, &state->flags);
	if (opendata->o_res.rflags & NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK)
		set_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags);

	dentry = opendata->dentry;
	if (d_really_is_negative(dentry)) {
		struct dentry *alias;
		d_drop(dentry);
		alias = d_exact_alias(dentry, state->inode);
		if (!alias)
			alias = d_splice_alias(igrab(state->inode), dentry);
		/* d_splice_alias() can't fail here - it's a non-directory */
		if (alias) {
			dput(ctx->dentry);
			ctx->dentry = dentry = alias;
		}
	}

	switch(opendata->o_arg.claim) {
	default:
		break;
	case NFS4_OPEN_CLAIM_NULL:
	case NFS4_OPEN_CLAIM_DELEGATE_CUR:
	case NFS4_OPEN_CLAIM_DELEGATE_PREV:
		if (!opendata->rpc_done)
			break;
		if (opendata->o_res.delegation_type != 0)
			dir_verifier = nfs_save_change_attribute(dir);
		nfs_set_verifier(dentry, dir_verifier);
	}

	/* Parse layoutget results before we check for access */
	pnfs_parse_lgopen(state->inode, opendata->lgp, ctx);

	ret = nfs4_opendata_access(sp->so_cred, opendata, state,
			acc_mode, flags);
	if (ret != 0)
		goto out;

	if (d_inode(dentry) == state->inode) {
		nfs_inode_attach_open_context(ctx);
		if (read_seqcount_retry(&sp->so_reclaim_seqcount, seq))
			nfs4_schedule_stateid_recovery(server, state);
	}

out:
	if (!opendata->cancelled)
		nfs4_sequence_free_slot(&opendata->o_res.seq_res);
	return ret;
}


static int nfs4_proc_symlink(struct inode *dir, struct dentry *dentry,
		struct page *page, unsigned int len, struct iattr *sattr)
{
	struct nfs4_exception exception = {
		.interruptible = true,
	};
	struct nfs4_label l, *label = NULL;
	int err;

	label = nfs4_label_init_security(dir, dentry, sattr, &l);

	do {
		err = _nfs4_proc_symlink(dir, dentry, page, len, sattr, label);
		trace_nfs4_symlink(dir, &dentry->d_name, err);
		err = nfs4_handle_exception(NFS_SERVER(dir), err,
				&exception);
	} while (exception.retry);

	nfs4_label_release_security(label);
	return err;
}


static int decode_threshold_hint(struct xdr_stream *xdr,
				  uint32_t *bitmap,
				  uint64_t *res,
				  uint32_t hint_bit)
{
	__be32 *p;

	*res = 0;
	if (likely(bitmap[0] & hint_bit)) {
		p = xdr_inline_decode(xdr, 8);
		if (unlikely(!p))
			return -EIO;
		xdr_decode_hyper(p, res);
	}
	return 0;
}


static int decode_attr_xattrsupport(struct xdr_stream *xdr, uint32_t *bitmap,
				    uint32_t *res)
{
	__be32 *p;

	*res = 0;
	if (unlikely(bitmap[2] & (FATTR4_WORD2_XATTR_SUPPORT - 1U)))
		return -EIO;
	if (likely(bitmap[2] & FATTR4_WORD2_XATTR_SUPPORT)) {
		p = xdr_inline_decode(xdr, 4);
		if (unlikely(!p))
			return -EIO;
		*res = be32_to_cpup(p);
		bitmap[2] &= ~FATTR4_WORD2_XATTR_SUPPORT;
	}
	dprintk("%s: XATTR support=%s\n", __func__,
		*res == 0 ? "false" : "true");
	return 0;
}


static void nfs4_exchange_id_release(void *data)
{
	struct nfs41_exchange_id_data *cdata =
					(struct nfs41_exchange_id_data *)data;

	nfs_put_client(cdata->args.client);
	kfree(cdata->res.impl_id);
	kfree(cdata->res.server_scope);
	kfree(cdata->res.server_owner);
	kfree(cdata);
}


int nfs4_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
		       bool plus)
{
	unsigned int savep;
	uint32_t bitmap[3] = {0};
	uint32_t len;
	uint64_t new_cookie;
	__be32 *p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EAGAIN;
	if (*p == xdr_zero) {
		p = xdr_inline_decode(xdr, 4);
		if (unlikely(!p))
			return -EAGAIN;
		if (*p == xdr_zero)
			return -EAGAIN;
		entry->eof = 1;
		return -EBADCOOKIE;
	}

	p = xdr_inline_decode(xdr, 12);
	if (unlikely(!p))
		return -EAGAIN;
	p = xdr_decode_hyper(p, &new_cookie);
	entry->len = be32_to_cpup(p);

	p = xdr_inline_decode(xdr, entry->len);
	if (unlikely(!p))
		return -EAGAIN;
	entry->name = (const char *) p;

	/*
	 * In case the server doesn't return an inode number,
	 * we fake one here.  (We don't use inode number 0,
	 * since glibc seems to choke on it...)
	 */
	entry->ino = 1;
	entry->fattr->valid = 0;

	if (decode_attr_bitmap(xdr, bitmap) < 0)
		return -EAGAIN;

	if (decode_attr_length(xdr, &len, &savep) < 0)
		return -EAGAIN;

	if (decode_getfattr_attrs(xdr, bitmap, entry->fattr, entry->fh,
			NULL, entry->label, entry->server) < 0)
		return -EAGAIN;
	if (entry->fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID)
		entry->ino = entry->fattr->mounted_on_fileid;
	else if (entry->fattr->valid & NFS_ATTR_FATTR_FILEID)
		entry->ino = entry->fattr->fileid;

	entry->d_type = DT_UNKNOWN;
	if (entry->fattr->valid & NFS_ATTR_FATTR_TYPE)
		entry->d_type = nfs_umode_to_dtype(entry->fattr->mode);

	entry->prev_cookie = entry->cookie;
	entry->cookie = new_cookie;

	return 0;
}


nfs4_async_handle_exception(struct rpc_task *task, struct nfs_server *server,
		int errorcode, struct nfs4_exception *exception)
{
	struct nfs_client *clp = server->nfs_client;
	int ret;

	ret = nfs4_do_handle_exception(server, errorcode, exception);
	if (exception->delay) {
		rpc_delay(task, nfs4_update_delay(&exception->timeout));
		goto out_retry;
	}
	if (exception->recovering) {
		rpc_sleep_on(&clp->cl_rpcwaitq, task, NULL);
		if (test_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) == 0)
			rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task);
		goto out_retry;
	}
	if (test_bit(NFS_MIG_FAILED, &server->mig_status))
		ret = -EIO;
	return ret;
out_retry:
	if (ret == 0) {
		exception->retry = 1;
		/*
		 * For NFS4ERR_MOVED, the client transport will need to
		 * be recomputed after migration recovery has completed.
		 */
		if (errorcode == -NFS4ERR_MOVED)
			rpc_task_release_transport(task);
	}
	return ret;
}


static int nfs4_xdr_dec_access(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
			       void *data)
{
	struct nfs4_accessres *res = data;
	struct compound_hdr hdr;
	int status;

	status = decode_compound_hdr(xdr, &hdr);
	if (status)
		goto out;
	status = decode_sequence(xdr, &res->seq_res, rqstp);
	if (status)
		goto out;
	status = decode_putfh(xdr);
	if (status != 0)
		goto out;
	status = decode_access(xdr, &res->supported, &res->access);
	if (status != 0)
		goto out;
	if (res->fattr)
		decode_getfattr(xdr, res->fattr, res->server);
out:
	return status;
}


static void nfs4_update_session(struct nfs4_session *session,
		struct nfs41_create_session_res *res)
{
	nfs4_copy_sessionid(&session->sess_id, &res->sessionid);
	/* Mark client id and session as being confirmed */
	session->clp->cl_exchange_flags |= EXCHGID4_FLAG_CONFIRMED_R;
	set_bit(NFS4_SESSION_ESTABLISHED, &session->session_state);
	session->flags = res->flags;
	memcpy(&session->fc_attrs, &res->fc_attrs, sizeof(session->fc_attrs));
	if (res->flags & SESSION4_BACK_CHAN)
		memcpy(&session->bc_attrs, &res->bc_attrs,
				sizeof(session->bc_attrs));
}


static int nfs4_do_open_reclaim(struct nfs_open_context *ctx, struct nfs4_state *state)
{
	struct nfs_server *server = NFS_SERVER(state->inode);
	struct nfs4_exception exception = { };
	int err;
	do {
		err = _nfs4_do_open_reclaim(ctx, state);
		trace_nfs4_open_reclaim(ctx, 0, err);
		if (nfs4_clear_cap_atomic_open_v1(server, err, &exception))
			continue;
		if (err != -NFS4ERR_DELAY)
			break;
		nfs4_handle_exception(server, err, &exception);
	} while (exception.retry);
	return err;
}


static int nfs4_write_done_cb(struct rpc_task *task,
			      struct nfs_pgio_header *hdr)
{
	struct inode *inode = hdr->inode;

	trace_nfs4_write(hdr, task->tk_status);
	if (task->tk_status < 0) {
		struct nfs4_exception exception = {
			.inode = hdr->inode,
			.state = hdr->args.context->state,
			.stateid = &hdr->args.stateid,
		};
		task->tk_status = nfs4_async_handle_exception(task,
				NFS_SERVER(inode), task->tk_status,
				&exception);
		if (exception.retry) {
			rpc_restart_call_prepare(task);
			return -EAGAIN;
		}
	}
	if (task->tk_status >= 0) {
		renew_lease(NFS_SERVER(inode), hdr->timestamp);
		nfs_writeback_update_inode(hdr);
	}
	return 0;
}


static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata)
{
	struct nfs4_delegreturndata *data = calldata;
	struct nfs4_exception exception = {
		.inode = data->inode,
		.stateid = &data->stateid,
	};

	if (!nfs4_sequence_done(task, &data->res.seq_res))
		return;

	trace_nfs4_delegreturn_exit(&data->args, &data->res, task->tk_status);

	/* Handle Layoutreturn errors */
	if (pnfs_roc_done(task, data->inode,
				&data->args.lr_args,
				&data->res.lr_res,
				&data->res.lr_ret) == -EAGAIN)
		goto out_restart;

	switch (task->tk_status) {
	case 0:
		renew_lease(data->res.server, data->timestamp);
		break;
	case -NFS4ERR_ADMIN_REVOKED:
	case -NFS4ERR_DELEG_REVOKED:
	case -NFS4ERR_EXPIRED:
		nfs4_free_revoked_stateid(data->res.server,
				data->args.stateid,
				task->tk_msg.rpc_cred);
		/* Fallthrough */
	case -NFS4ERR_BAD_STATEID:
	case -NFS4ERR_STALE_STATEID:
	case -ETIMEDOUT:
		task->tk_status = 0;
		break;
	case -NFS4ERR_OLD_STATEID:
		if (!nfs4_refresh_delegation_stateid(&data->stateid, data->inode))
			nfs4_stateid_seqid_inc(&data->stateid);
		if (data->args.bitmask) {
			data->args.bitmask = NULL;
			data->res.fattr = NULL;
		}
		goto out_restart;
	case -NFS4ERR_ACCESS:
		if (data->args.bitmask) {
			data->args.bitmask = NULL;
			data->res.fattr = NULL;
			goto out_restart;
		}
		/* Fallthrough */
	default:
		task->tk_status = nfs4_async_handle_exception(task,
				data->res.server, task->tk_status,
				&exception);
		if (exception.retry)
			goto out_restart;
	}
	nfs_delegation_mark_returned(data->inode, data->args.stateid);
	data->rpc_status = task->tk_status;
	return;
out_restart:
	task->tk_status = 0;
	rpc_restart_call_prepare(task);
}


static void nfs41_call_sync_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs4_call_sync_data *data = calldata;

	dprintk("--> %s data->seq_server %p\n", __func__, data->seq_server);

	nfs4_setup_sequence(data->seq_server->nfs_client,
			    data->seq_args, data->seq_res, task);
}


static void nfs4_xdr_enc_create(struct rpc_rqst *req, struct xdr_stream *xdr,
				const void *data)
{
	const struct nfs4_create_arg *args = data;
	struct compound_hdr hdr = {
		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
	};

	encode_compound_hdr(xdr, req, &hdr);
	encode_sequence(xdr, &args->seq_args, &hdr);
	encode_putfh(xdr, args->dir_fh, &hdr);
	encode_create(xdr, args, &hdr);
	encode_getfh(xdr, &hdr);
	encode_getfattr(xdr, args->bitmask, &hdr);
	encode_nops(&hdr);
}


static void encode_create(struct xdr_stream *xdr, const struct nfs4_create_arg *create, struct compound_hdr *hdr)
{
	__be32 *p;

	encode_op_hdr(xdr, OP_CREATE, decode_create_maxsz, hdr);
	encode_uint32(xdr, create->ftype);

	switch (create->ftype) {
	case NF4LNK:
		p = reserve_space(xdr, 4);
		*p = cpu_to_be32(create->u.symlink.len);
		xdr_write_pages(xdr, create->u.symlink.pages, 0,
				create->u.symlink.len);
		xdr->buf->flags |= XDRBUF_WRITE;
		break;

	case NF4BLK: case NF4CHR:
		p = reserve_space(xdr, 8);
		*p++ = cpu_to_be32(create->u.device.specdata1);
		*p = cpu_to_be32(create->u.device.specdata2);
		break;

	default:
		break;
	}

	encode_string(xdr, create->name->len, create->name->name);
	encode_attrs(xdr, create->attrs, create->label, &create->umask,
			create->server, create->server->attr_bitmask);
}


static int nfs4_xdr_dec_test_stateid(struct rpc_rqst *rqstp,
				     struct xdr_stream *xdr,
				     void *data)
{
	struct nfs41_test_stateid_res *res = data;
	struct compound_hdr hdr;
	int status;

	status = decode_compound_hdr(xdr, &hdr);
	if (status)
		goto out;
	status = decode_sequence(xdr, &res->seq_res, rqstp);
	if (status)
		goto out;
	status = decode_test_stateid(xdr, res);
out:
	return status;
}


static int decode_rw_delegation(struct xdr_stream *xdr,
		uint32_t delegation_type,
		struct nfs_openres *res)
{
	__be32 *p;
	int status;

	status = decode_delegation_stateid(xdr, &res->delegation);
	if (unlikely(status))
		return status;
	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	res->do_recall = be32_to_cpup(p);

	switch (delegation_type) {
	case NFS4_OPEN_DELEGATE_READ:
		res->delegation_type = FMODE_READ;
		break;
	case NFS4_OPEN_DELEGATE_WRITE:
		res->delegation_type = FMODE_WRITE|FMODE_READ;
		if (decode_space_limit(xdr, &res->pagemod_limit) < 0)
				return -EIO;
	}
	return decode_ace(xdr, NULL);
}


static void nfs4_xdr_enc_readlink(struct rpc_rqst *req, struct xdr_stream *xdr,
				  const void *data)
{
	const struct nfs4_readlink *args = data;
	struct compound_hdr hdr = {
		.minorversion = nfs4_xdr_minorversion(&args->seq_args),
	};

	encode_compound_hdr(xdr, req, &hdr);
	encode_sequence(xdr, &args->seq_args, &hdr);
	encode_putfh(xdr, args->fh, &hdr);
	encode_readlink(xdr, args, req, &hdr);

	rpc_prepare_reply_pages(req, args->pages, args->pgbase,
				args->pglen, hdr.replen);
	encode_nops(&hdr);
}


int nfs4_setup_sequence(struct nfs_client *client,
			struct nfs4_sequence_args *args,
			struct nfs4_sequence_res *res,
			struct rpc_task *task)
{
	struct nfs4_session *session = nfs4_get_session(client);
	struct nfs4_slot_table *tbl  = client->cl_slot_tbl;
	struct nfs4_slot *slot;

	/* slot already allocated? */
	if (res->sr_slot != NULL)
		goto out_start;

	if (session)
		tbl = &session->fc_slot_table;

	spin_lock(&tbl->slot_tbl_lock);
	/* The state manager will wait until the slot table is empty */
	if (nfs4_slot_tbl_draining(tbl) && !args->sa_privileged)
		goto out_sleep;

	slot = nfs4_alloc_slot(tbl);
	if (IS_ERR(slot)) {
		if (slot == ERR_PTR(-ENOMEM))
			goto out_sleep_timeout;
		goto out_sleep;
	}
	spin_unlock(&tbl->slot_tbl_lock);

	nfs4_sequence_attach_slot(args, res, slot);

	trace_nfs4_setup_sequence(session, args);
out_start:
	nfs41_sequence_res_init(res);
	rpc_call_start(task);
	return 0;
out_sleep_timeout:
	/* Try again in 1/4 second */
	if (args->sa_privileged)
		rpc_sleep_on_priority_timeout(&tbl->slot_tbl_waitq, task,
				jiffies + (HZ >> 2), RPC_PRIORITY_PRIVILEGED);
	else
		rpc_sleep_on_timeout(&tbl->slot_tbl_waitq, task,
				NULL, jiffies + (HZ >> 2));
	spin_unlock(&tbl->slot_tbl_lock);
	return -EAGAIN;
out_sleep:
	if (args->sa_privileged)
		rpc_sleep_on_priority(&tbl->slot_tbl_waitq, task,
				RPC_PRIORITY_PRIVILEGED);
	else
		rpc_sleep_on(&tbl->slot_tbl_waitq, task, NULL);
	spin_unlock(&tbl->slot_tbl_lock);
	return -EAGAIN;
}


static int nfs4_xdr_dec_close(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
			      void *data)
{
	struct nfs_closeres *res = data;
	struct compound_hdr hdr;
	int status;

	status = decode_compound_hdr(xdr, &hdr);
	if (status)
		goto out;
	status = decode_sequence(xdr, &res->seq_res, rqstp);
	if (status)
		goto out;
	status = decode_putfh(xdr);
	if (status)
		goto out;
	if (res->lr_res) {
		status = decode_layoutreturn(xdr, res->lr_res);
		res->lr_ret = status;
		if (status)
			goto out;
	}
	if (res->fattr != NULL) {
		status = decode_getfattr(xdr, res->fattr, res->server);
		if (status != 0)
			goto out;
	}
	status = decode_close(xdr, res);
out:
	return status;
}


static int decode_getdeviceinfo(struct xdr_stream *xdr,
				struct nfs4_getdeviceinfo_res *res)
{
	struct pnfs_device *pdev = res->pdev;
	__be32 *p;
	uint32_t len, type;
	int status;

	status = decode_op_hdr(xdr, OP_GETDEVICEINFO);
	if (status) {
		if (status == -ETOOSMALL) {
			p = xdr_inline_decode(xdr, 4);
			if (unlikely(!p))
				return -EIO;
			pdev->mincount = be32_to_cpup(p);
			dprintk("%s: Min count too small. mincnt = %u\n",
				__func__, pdev->mincount);
		}
		return status;
	}

	p = xdr_inline_decode(xdr, 8);
	if (unlikely(!p))
		return -EIO;
	type = be32_to_cpup(p++);
	if (type != pdev->layout_type) {
		dprintk("%s: layout mismatch req: %u pdev: %u\n",
			__func__, pdev->layout_type, type);
		return -EINVAL;
	}
	/*
	 * Get the length of the opaque device_addr4. xdr_read_pages places
	 * the opaque device_addr4 in the xdr_buf->pages (pnfs_device->pages)
	 * and places the remaining xdr data in xdr_buf->tail
	 */
	pdev->mincount = be32_to_cpup(p);
	if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount)
		return -EIO;

	/* Parse notification bitmap, verifying that it is zero. */
	p = xdr_inline_decode(xdr, 4);
	if (unlikely(!p))
		return -EIO;
	len = be32_to_cpup(p);
	if (len) {
		uint32_t i;

		p = xdr_inline_decode(xdr, 4 * len);
		if (unlikely(!p))
			return -EIO;

		res->notification = be32_to_cpup(p++);
		for (i = 1; i < len; i++) {
			if (be32_to_cpup(p++)) {
				dprintk("%s: unsupported notification\n",
					__func__);
				return -EIO;
			}
		}
	}
	return 0;
}


std::string Utility::localPathFromFilePath(const absl::string_view& file_path) {
  if (file_path.size() >= 3 && file_path[1] == ':' && file_path[2] == '/' &&
      std::isalpha(file_path[0])) {
    return std::string(file_path);
  }
  return absl::StrCat("/", file_path);
}


bool operator==(const Component& other) const { return begin == other.begin && len == other.len; }


absl::string_view hostAndPort() { return host_and_port_; }


initializeAndValidateOptions(const envoy::config::core::v3::Http2ProtocolOptions& options,
                             bool hcm_stream_error_set,
                             const Protobuf::BoolValue& hcm_stream_error) {
  auto ret = initializeAndValidateOptions(options);
  if (Runtime::runtimeFeatureEnabled(
          "envoy.reloadable_features.hcm_stream_error_on_invalid_message") &&
      !options.has_override_stream_error_on_invalid_http_message() && hcm_stream_error_set) {
    ret.mutable_override_stream_error_on_invalid_http_message()->set_value(
        hcm_stream_error.value());
  }
  return ret;
}


absl::string_view pathAndQueryParams() { return path_and_query_params_; }


TEST(HttpUtility, RateLimitedGrpcStatus) {
  MockStreamDecoderFilterCallbacks callbacks;

  EXPECT_CALL(callbacks, streamInfo()).Times(testing::AnyNumber());
  EXPECT_CALL(callbacks, encodeHeaders_(_, true))
      .WillOnce(Invoke([&](const ResponseHeaderMap& headers, bool) -> void {
        EXPECT_NE(headers.GrpcStatus(), nullptr);
        EXPECT_EQ(headers.getGrpcStatusValue(),
                  std::to_string(enumToInt(Grpc::Status::WellKnownGrpcStatus::Unavailable)));
      }));
  Utility::sendLocalReply(
      false, callbacks,
      Utility::LocalReplyData{true, Http::Code::TooManyRequests, "", absl::nullopt, false});

  EXPECT_CALL(callbacks, encodeHeaders_(_, true))
      .WillOnce(Invoke([&](const ResponseHeaderMap& headers, bool) -> void {
        EXPECT_NE(headers.GrpcStatus(), nullptr);
        EXPECT_EQ(headers.getGrpcStatusValue(),
                  std::to_string(enumToInt(Grpc::Status::WellKnownGrpcStatus::ResourceExhausted)));
      }));
  Utility::sendLocalReply(
      false, callbacks,
      Utility::LocalReplyData{true, Http::Code::TooManyRequests, "",
                              absl::make_optional<Grpc::Status::GrpcStatus>(
                                  Grpc::Status::WellKnownGrpcStatus::ResourceExhausted),
                              false});
}


    HeaderLocationSpec(const Http::LowerCaseString& header, const std::string& value_prefix)
        : header_(header), value_prefix_(value_prefix) {}


TEST(HeaderMapImplTest, AddCopy) {
  TestRequestHeaderMapImpl headers;

  // Start with a string value.
  std::unique_ptr<LowerCaseString> lcKeyPtr(new LowerCaseString("hello"));
  headers.addCopy(*lcKeyPtr, "world");

  const HeaderString& value = headers.get(*lcKeyPtr)->value();

  EXPECT_EQ("world", value.getStringView());
  EXPECT_EQ(5UL, value.size());

  lcKeyPtr.reset();

  const HeaderString& value2 = headers.get(LowerCaseString("hello"))->value();

  EXPECT_EQ("world", value2.getStringView());
  EXPECT_EQ(5UL, value2.size());
  EXPECT_EQ(value.getStringView(), value2.getStringView());
  EXPECT_EQ(1UL, headers.size());

  // Repeat with an int value.
  //
  // addReferenceKey and addCopy can both add multiple instances of a
  // given header, so we need to delete the old "hello" header.
  // Test that removing will return 0 byte size.
  EXPECT_EQ(1UL, headers.remove(LowerCaseString("hello")));
  EXPECT_EQ(headers.byteSize(), 0);

  // Build "hello" with string concatenation to make it unlikely that the
  // compiler is just reusing the same string constant for everything.
  lcKeyPtr = std::make_unique<LowerCaseString>(std::string("he") + "llo");
  EXPECT_STREQ("hello", lcKeyPtr->get().c_str());

  headers.addCopy(*lcKeyPtr, 42);

  const HeaderString& value3 = headers.get(*lcKeyPtr)->value();

  EXPECT_EQ("42", value3.getStringView());
  EXPECT_EQ(2UL, value3.size());

  lcKeyPtr.reset();

  const HeaderString& value4 = headers.get(LowerCaseString("hello"))->value();

  EXPECT_EQ("42", value4.getStringView());
  EXPECT_EQ(2UL, value4.size());
  EXPECT_EQ(1UL, headers.size());

  // Here, again, we'll build yet another key string.
  LowerCaseString lcKey3(std::string("he") + "ll" + "o");
  EXPECT_STREQ("hello", lcKey3.get().c_str());

  EXPECT_EQ("42", headers.get(lcKey3)->value().getStringView());
  EXPECT_EQ(2UL, headers.get(lcKey3)->value().size());

  LowerCaseString envoy_retry_on("x-envoy-retry-on");
  headers.addCopy(envoy_retry_on, "max-age=1345");
  EXPECT_EQ("max-age=1345", headers.get(envoy_retry_on)->value().getStringView());
  EXPECT_EQ("max-age=1345", headers.getEnvoyRetryOnValue());
  headers.addCopy(envoy_retry_on, "public");
  EXPECT_EQ("max-age=1345,public", headers.get(envoy_retry_on)->value().getStringView());
  headers.addCopy(envoy_retry_on, "");
  EXPECT_EQ("max-age=1345,public", headers.get(envoy_retry_on)->value().getStringView());
  headers.addCopy(envoy_retry_on, 123);
  EXPECT_EQ("max-age=1345,public,123", headers.get(envoy_retry_on)->value().getStringView());
  headers.addCopy(envoy_retry_on, std::numeric_limits<uint64_t>::max());
  EXPECT_EQ("max-age=1345,public,123,18446744073709551615",
            headers.get(envoy_retry_on)->value().getStringView());
}


HeaderToMetadataFilter::HeaderToMetadataFilter(const ConfigSharedPtr config) : config_(config) {}


TEST_F(LuaStreamInfoWrapperTest, ReturnCurrentProtocol) {
  expectToPrintCurrentProtocol(Http::Protocol::Http10);
  expectToPrintCurrentProtocol(Http::Protocol::Http11);
  expectToPrintCurrentProtocol(Http::Protocol::Http2);
}


HeaderUtility::getAllOfHeaderAsString(const HeaderMap& headers, const Http::LowerCaseString& key) {
  GetAllOfHeaderAsStringResult result;
  const auto header_value = headers.getAll(key);

  if (header_value.empty()) {
    // Empty for clarity. Avoid handling the empty case in the block below if the runtime feature
    // is disabled.
  } else if (header_value.size() == 1 ||
             !Runtime::runtimeFeatureEnabled(
                 "envoy.reloadable_features.http_match_on_all_headers")) {
    result.result_ = header_value[0]->value().getStringView();
  } else {
    // In this case we concatenate all found headers using a ',' delimiter before performing the
    // final match. We use an InlinedVector of absl::string_view to invoke the optimized join
    // algorithm. This requires a copying phase before we invoke join. The 3 used as the inline
    // size has been arbitrarily chosen.
    // TODO(mattklein123): Do we need to normalize any whitespace here?
    absl::InlinedVector<absl::string_view, 3> string_view_vector;
    string_view_vector.reserve(header_value.size());
    for (size_t i = 0; i < header_value.size(); i++) {
      string_view_vector.push_back(header_value[i]->value().getStringView());
    }
    result.result_backing_string_ = absl::StrJoin(string_view_vector, ",");
  }

  return result;
}


int HeaderMapWrapper::luaGet(lua_State* state) {
  const char* key = luaL_checkstring(state, 2);
  const auto value =
      Http::HeaderUtility::getAllOfHeaderAsString(headers_, Http::LowerCaseString(key));
  if (value.result().has_value()) {
    lua_pushlstring(state, value.result().value().data(), value.result().value().length());
    return 1;
  } else {
    return 0;
  }
}


  RegisterCustomInlineHeader(const LowerCaseString& header)
      : handle_(CustomInlineHeaderRegistry::registerInlineHeader<type>(header)) {}


StreamInfo::StreamInfo& DynamicMetadataMapWrapper::streamInfo() { return parent_.stream_info_; }


size_t size() const { return result_.size(); }


  LowerCaseString(LowerCaseString&& rhs) noexcept : string_(std::move(rhs.string_)) {
    ASSERT(valid());
  }


ConnectionWrapper(const StreamInfo::StreamInfo& info) : info_(info) {}


  void setInline(Handle handle, absl::string_view value) override {
    ASSERT(handle.it_->second < inlineHeadersSize());
    HeaderEntry& entry = maybeCreateInline(&inlineHeaders()[handle.it_->second], handle.it_->first);
    updateSize(entry.value().size(), value.size());
    entry.value().setCopy(value);
  }


void HeaderToMetadataFilter::setDecoderFilterCallbacks(
    Http::StreamDecoderFilterCallbacks& callbacks) {
  decoder_callbacks_ = &callbacks;
}


HeaderString::HeaderString(const LowerCaseString& ref_value)
    : buffer_(absl::string_view(ref_value.get().c_str(), ref_value.get().size())) {
  ASSERT(valid());
}


void HeaderMapImpl::addCopy(const LowerCaseString& key, absl::string_view value) {
  // In the case that the header is appended, we will perform a needless copy of the key and value.
  // This is done on purpose to keep the code simple and should be rare.
  HeaderString new_key;
  new_key.setCopy(key.get());
  HeaderString new_value;
  new_value.setCopy(value);
  insertByKey(std::move(new_key), std::move(new_value));
  ASSERT(new_key.empty());   // NOLINT(bugprone-use-after-move)
  ASSERT(new_value.empty()); // NOLINT(bugprone-use-after-move)
}


  JwtHeaderLocation(const std::string& token, const absl::node_hash_set<std::string>& issuers,
                    const LowerCaseString& header)
      : JwtLocationBase(token, issuers), header_(header) {}


HeaderUtility::HeaderData::HeaderData(const envoy::config::route::v3::HeaderMatcher& config)
    : name_(config.name()), invert_match_(config.invert_match()) {
  switch (config.header_match_specifier_case()) {
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kExactMatch:
    header_match_type_ = HeaderMatchType::Value;
    value_ = config.exact_match();
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::
      kHiddenEnvoyDeprecatedRegexMatch:
    header_match_type_ = HeaderMatchType::Regex;
    regex_ = Regex::Utility::parseStdRegexAsCompiledMatcher(
        config.hidden_envoy_deprecated_regex_match());
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kSafeRegexMatch:
    header_match_type_ = HeaderMatchType::Regex;
    regex_ = Regex::Utility::parseRegex(config.safe_regex_match());
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kRangeMatch:
    header_match_type_ = HeaderMatchType::Range;
    range_.set_start(config.range_match().start());
    range_.set_end(config.range_match().end());
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kPresentMatch:
    header_match_type_ = HeaderMatchType::Present;
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kPrefixMatch:
    header_match_type_ = HeaderMatchType::Prefix;
    value_ = config.prefix_match();
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kSuffixMatch:
    header_match_type_ = HeaderMatchType::Suffix;
    value_ = config.suffix_match();
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::kContainsMatch:
    header_match_type_ = HeaderMatchType::Contains;
    value_ = config.contains_match();
    break;
  case envoy::config::route::v3::HeaderMatcher::HeaderMatchSpecifierCase::
      HEADER_MATCH_SPECIFIER_NOT_SET:
    FALLTHRU;
  default:
    header_match_type_ = HeaderMatchType::Present;
    break;
  }
}


const std::string& get() const { return string_; }


const std::string& backingString() const { return result_backing_string_; }


TEST_P(RBACIntegrationTest, HeaderMatchConditionDuplicateHeaderNoMatch) {
  config_helper_.addFilter(fmt::format(RBAC_CONFIG_HEADER_MATCH_CONDITION, "yyy"));
  initialize();

  codec_client_ = makeHttpConnection(lookupPort("http"));

  auto response = codec_client_->makeRequestWithBody(
      Http::TestRequestHeaderMapImpl{
          {":method", "POST"},
          {":path", "/path"},
          {":scheme", "http"},
          {":authority", "host"},
          {"xxx", "yyy"},
          {"xxx", "zzz"},
      },
      1024);
  response->waitForEndStream();
  ASSERT_TRUE(response->complete());
  EXPECT_EQ("403", response->headers().getStatusValue());
}


  void addReference(const LowerCaseString& key, absl::string_view value) override {
    HeaderMapImpl::addReference(key, value);
  }


TEST(MatchHeadersTest, HeaderSafeRegexMatch) {
  TestRequestHeaderMapImpl matching_headers{{"match-header", "123"}};
  TestRequestHeaderMapImpl unmatching_headers{{"match-header", "1234"},
                                              {"match-header", "123.456"}};
  const std::string yaml = R"EOF(
name: match-header
safe_regex_match:
  google_re2: {}
  regex: \d{3}
  )EOF";

  std::vector<HeaderUtility::HeaderDataPtr> header_data;
  header_data.push_back(
      std::make_unique<HeaderUtility::HeaderData>(parseHeaderMatcherFromYaml(yaml)));
  EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data));
  EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data));
}


TEST_F(HeaderToMetadataTest, RejectBothCookieHeader) {
  const std::string config = R"EOF(
request_rules:
  - header: x-something
    cookie: something-else
    on_header_present:
      key: something
      value: else
      type: STRING
    remove: false

)EOF";
  auto expected = "Cannot specify both header and cookie";
  EXPECT_THROW_WITH_MESSAGE(initializeFilter(config), EnvoyException, expected);
}


void HeaderMapImpl::addReference(const LowerCaseString& key, absl::string_view value) {
  HeaderString ref_key(key);
  HeaderString ref_value(value);
  insertByKey(std::move(ref_key), std::move(ref_value));
}


const HeaderEntry* operator[](size_t i) const { return result_[i]; }


  void setReferenceInline(Handle handle, absl::string_view value) override {
    header_map_->setReferenceInline(handle, value);
    header_map_->verifyByteSizeInternalForTest();
  }


static inline bool validHeaderString(absl::string_view s) {
  // If you modify this list of illegal embedded characters you will probably
  // want to change header_map_fuzz_impl_test at the same time.
  for (const char c : s) {
    switch (c) {
    case '\0':
      FALLTHRU;
    case '\r':
      FALLTHRU;
    case '\n':
      return false;
    default:
      continue;
    }
  }
  return true;
}


createHeaderMap(const std::initializer_list<std::pair<LowerCaseString, std::string>>& values) {
  auto new_header_map = T::create();
  HeaderMapImpl::initFromInitList(*new_header_map, values.begin(), values.end());
  return new_header_map;
}


const HeaderEntry* HeaderMapImpl::get(const LowerCaseString& key) const {
  const auto result = getAll(key);
  return result.empty() ? nullptr : result[0];
}


TEST_F(ExtractorTest, TestWrongHeaderToken) {
  auto headers = TestRequestHeaderMapImpl{{"wrong-token-header", "jwt_token"}};
  auto tokens = extractor_->extract(headers);
  EXPECT_EQ(tokens.size(), 0);
}


  static std::unique_ptr<RequestTrailerMapImpl> create() {
    return std::unique_ptr<RequestTrailerMapImpl>(new (inlineHeadersSize())
                                                      RequestTrailerMapImpl());
  }


TEST(MatchHeadersTest, HeaderSuffixInverseMatch) {
  TestRequestHeaderMapImpl unmatching_headers{{"match-header", "123value"}};
  TestRequestHeaderMapImpl matching_headers{{"match-header", "value123"}};

  const std::string yaml = R"EOF(
name: match-header
suffix_match: value
invert_match: true
  )EOF";

  std::vector<HeaderUtility::HeaderDataPtr> header_data;
  header_data.push_back(
      std::make_unique<HeaderUtility::HeaderData>(parseHeaderMatcherFromYaml(yaml)));
  EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data));
  EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data));
}


  void iterateReverse(HeaderMap::ConstIterateCb cb) const override {
    header_map_->iterateReverse(cb);
  }


TEST_F(LuaHeaderMapWrapperTest, Methods) {
  const std::string SCRIPT{R"EOF(
    function callMe(object)
      object:add("HELLO", "WORLD")
      testPrint(object:get("hELLo"))

      object:add("header1", "")
      object:add("header2", "foo")

      for key, value in pairs(object) do
        testPrint(string.format("'%s' '%s'", key, value))
      end

      object:remove("header1")
      for key, value in pairs(object) do
        testPrint(string.format("'%s' '%s'", key, value))
      end

      object:add("header3", "foo")
      object:add("header3", "bar")
      testPrint(object:get("header3"))
    end
  )EOF"};

  InSequence s;
  setup(SCRIPT);

  Http::TestRequestHeaderMapImpl headers;
  HeaderMapWrapper::create(coroutine_->luaState(), headers, []() { return true; });
  EXPECT_CALL(printer_, testPrint("WORLD"));
  EXPECT_CALL(printer_, testPrint("'hello' 'WORLD'"));
  EXPECT_CALL(printer_, testPrint("'header1' ''"));
  EXPECT_CALL(printer_, testPrint("'header2' 'foo'"));
  EXPECT_CALL(printer_, testPrint("'hello' 'WORLD'"));
  EXPECT_CALL(printer_, testPrint("'header2' 'foo'"));
  EXPECT_CALL(printer_, testPrint("foo,bar"));
  start("callMe");
}


TEST(HeaderMapImplTest, SetCopyNewBehavior) {
  TestRequestHeaderMapImpl headers;
  LowerCaseString foo("hello");
  headers.setCopy(foo, "world");
  EXPECT_EQ("world", headers.get(foo)->value().getStringView());

  // Overwrite value.
  headers.setCopy(foo, "monde");
  EXPECT_EQ("monde", headers.get(foo)->value().getStringView());

  // Add another foo header.
  headers.addCopy(foo, "monde2");
  EXPECT_EQ(headers.size(), 2);

  // The foo header is overridden.
  headers.setCopy(foo, "override-monde");
  EXPECT_EQ(headers.size(), 1);

  HeaderAndValueCb cb;

  InSequence seq;
  EXPECT_CALL(cb, Call("hello", "override-monde"));
  headers.iterate(cb.asIterateCb());

  // Test setting an empty string and then overriding.
  EXPECT_EQ(1UL, headers.remove(foo));
  EXPECT_EQ(headers.size(), 0);
  const std::string empty;
  headers.setCopy(foo, empty);
  EXPECT_EQ(headers.size(), 1);
  headers.setCopy(foo, "not-empty");
  EXPECT_EQ(headers.get(foo)->value().getStringView(), "not-empty");

  // Use setCopy with inline headers both indirectly and directly.
  headers.clear();
  EXPECT_EQ(headers.size(), 0);
  headers.setCopy(Headers::get().Path, "/");
  EXPECT_EQ(headers.size(), 1);
  EXPECT_EQ(headers.getPathValue(), "/");
  headers.setPath("/foo");
  EXPECT_EQ(headers.size(), 1);
  EXPECT_EQ(headers.getPathValue(), "/foo");
}


  size_t removeInline(Handle handle) override {
    const size_t rc = header_map_->removeInline(handle);
    header_map_->verifyByteSizeInternalForTest();
    return rc;
  }


  TestHeaderMapImplBase(const std::initializer_list<std::pair<std::string, std::string>>& values) {
    for (auto& value : values) {
      header_map_->addCopy(LowerCaseString(value.first), value.second);
    }
    header_map_->verifyByteSizeInternalForTest();
  }


  void setup(const std::string& script) override {
    Filters::Common::Lua::LuaWrappersTestBase<StreamInfoWrapper>::setup(script);
    state_->registerType<DynamicMetadataMapWrapper>();
    state_->registerType<DynamicMetadataMapIterator>();
  }


  decodeResources(const Protobuf::RepeatedPtrField<ProtobufWkt::Any>& resources,
                  const std::string& version, const std::string& name_field = "name") {
    TestOpaqueResourceDecoderImpl<MessageType> resource_decoder(name_field);
    return Config::DecodedResourcesWrapper(resource_decoder, resources, version);
  }


TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBase64UrlTest) {
  const std::string response_config_yaml = R"EOF(
response_rules:
  - header: x-authenticated
    on_header_present:
      key: auth
      type: PROTOBUF_VALUE
      encode: BASE64
)EOF";
  initializeFilter(response_config_yaml);

  ProtobufWkt::Value value;
  auto* s = value.mutable_struct_value();

  ProtobufWkt::Value v;
  v.set_string_value("blafoo");
  (*s->mutable_fields())["k1"] = v;
  v.set_number_value(2019.07);
  (*s->mutable_fields())["k2"] = v;
  v.set_bool_value(true);
  (*s->mutable_fields())["k3"] = v;

  std::string data;
  ASSERT_TRUE(value.SerializeToString(&data));
  const auto encoded = Base64::encode(data.c_str(), data.size());
  Http::TestResponseHeaderMapImpl incoming_headers{{"x-authenticated", encoded}};
  std::map<std::string, ProtobufWkt::Value> expected = {{"auth", value}};

  EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
  EXPECT_CALL(req_info_,
              setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEqValue(expected)));
  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
}


  void setReferenceKey(const LowerCaseString& key, absl::string_view value) override {
    header_map_->setReferenceKey(key, value);
  }


Config::Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config,
               const bool per_route) {
  request_set_ = Config::configToVector(config.request_rules(), request_rules_);
  response_set_ = Config::configToVector(config.response_rules(), response_rules_);

  // Note: empty configs are fine for the global config, which would be the case for enabling
  //       the filter globally without rules and then applying them at the virtual host or
  //       route level. At the virtual or route level, it makes no sense to have an empty
  //       config so we throw an error.
  if (per_route && !response_set_ && !request_set_) {
    throw EnvoyException("header_to_metadata_filter: Per filter configs must at least specify "
                         "either request or response rules");
  }
}


TEST_F(ExtractorTest, TestDefaultHeaderLocationWithValidJWT) {
  auto headers =
      TestRequestHeaderMapImpl{{absl::StrCat("Authorization"), absl::StrCat("Bearer ", GoodToken)}};
  auto tokens = extractor_->extract(headers);
  EXPECT_EQ(tokens.size(), 1);

  // Only the issue1 is using default header location.
  EXPECT_EQ(tokens[0]->token(), GoodToken);
  EXPECT_TRUE(tokens[0]->isIssuerSpecified("issuer1"));
}


absl::string_view ExtractorImpl::extractJWT(absl::string_view value_str,
                                            absl::string_view::size_type after) const {
  const auto starting = value_str.find_first_of(ConstantBase64UrlEncodingCharsPlusDot, after);
  if (starting == value_str.npos) {
    return value_str;
  }
  // There should be two dots (periods; 0x2e) inside the string, but we don't verify that here
  auto ending = value_str.find_first_not_of(ConstantBase64UrlEncodingCharsPlusDot, starting);
  if (ending == value_str.npos) { // Base64Url-encoded string occupies the rest of the line
    return value_str.substr(starting);
  }
  return value_str.substr(starting, ending - starting);
}


MATCHER_P(ProtoEqIgnoreRepeatedFieldOrdering, expected, "") {
  const bool equal =
      TestUtility::protoEqual(arg, expected, /*ignore_repeated_field_ordering=*/true);
  if (!equal) {
    *result_listener << "\n"
                     << TestUtility::addLeftAndRightPadding("Expected proto:") << "\n"
                     << expected.DebugString()
                     << TestUtility::addLeftAndRightPadding("is not equal to actual proto:") << "\n"
                     << arg.DebugString()
                     << TestUtility::addLeftAndRightPadding("") // line full of padding
                     << "\n";
  }
  return equal;
}


  static Config::DecodedResourcesWrapper decodeResources(
      const Protobuf::RepeatedPtrField<envoy::service::discovery::v3::Resource>& resources,
      const std::string& name_field = "name") {
    Config::DecodedResourcesWrapper decoded_resources;
    TestOpaqueResourceDecoderImpl<MessageType> resource_decoder(name_field);
    for (const auto& resource : resources) {
      decoded_resources.owned_resources_.emplace_back(
          new Config::DecodedResourceImpl(resource_decoder, resource));
      decoded_resources.refvec_.emplace_back(*decoded_resources.owned_resources_.back());
    }
    return decoded_resources;
  }


TEST_F(HeaderToMetadataTest, RegexSubstitution) {
  const std::string config = R"EOF(
request_rules:
  - header: :path
    on_header_present:
      metadata_namespace: envoy.lb
      key: cluster
      regex_value_rewrite:
        pattern:
          google_re2: {}
          regex: "^/(cluster[\\d\\w-]+)/?.*$"
        substitution: "\\1"
)EOF";
  initializeFilter(config);

  // Match with additional path elements.
  {
    Http::TestRequestHeaderMapImpl headers{{":path", "/cluster-prod-001/x/y"}};
    std::map<std::string, std::string> expected = {{"cluster", "cluster-prod-001"}};

    EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
    EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected)));
    EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false));
  }

  // Match with no additional path elements.
  {
    Http::TestRequestHeaderMapImpl headers{{":path", "/cluster-prod-001"}};
    std::map<std::string, std::string> expected = {{"cluster", "cluster-prod-001"}};

    EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
    EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected)));
    EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false));
  }

  // No match.
  {
    Http::TestRequestHeaderMapImpl headers{{":path", "/foo"}};
    std::map<std::string, std::string> expected = {{"cluster", "/foo"}};

    EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
    EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected)));
    EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false));
  }

  // No match with additional path elements.
  {
    Http::TestRequestHeaderMapImpl headers{{":path", "/foo/bar?x=2"}};
    std::map<std::string, std::string> expected = {{"cluster", "/foo/bar?x=2"}};

    EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
    EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected)));
    EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false));
  }
}


bool HeaderMapImpl::operator==(const HeaderMap& rhs) const {
  if (size() != rhs.size()) {
    return false;
  }

  std::vector<std::pair<absl::string_view, absl::string_view>> rhs_headers;
  rhs_headers.reserve(rhs.size());
  rhs.iterate(collectAllHeaders(&rhs_headers));

  auto i = headers_.begin();
  auto j = rhs_headers.begin();
  for (; i != headers_.end(); ++i, ++j) {
    if (i->key() != j->first || i->value() != j->second) {
      return false;
    }
  }

  return true;
}


RequestHeaderMapImpl() { clearInline(); }


checkoldpid(char* pidfile, int inchroot)
{
	pid_t old;
	if((old = readpid(pidfile)) != -1) {
		/* see if it is still alive */
		if(kill(old, 0) == 0 || errno == EPERM)
			log_warn("unbound is already running as pid %u.",
				(unsigned)old);
		else	if(inchroot)
			log_warn("did not exit gracefully last time (%u)",
				(unsigned)old);
	}
}


static void BrotliStoreHuffmanTreeToBitMask(
    const size_t huffman_tree_size, const uint8_t* huffman_tree,
    const uint8_t* huffman_tree_extra_bits, const uint8_t* code_length_bitdepth,
    const uint16_t* code_length_bitdepth_symbols,
    size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage) {
  size_t i;
  for (i = 0; i < huffman_tree_size; ++i) {
    size_t ix = huffman_tree[i];
    BrotliWriteBits(code_length_bitdepth[ix], code_length_bitdepth_symbols[ix],
                    storage_ix, storage);
    /* Extra bits */
    switch (ix) {
      case BROTLI_REPEAT_PREVIOUS_CODE_LENGTH:
        BrotliWriteBits(2, huffman_tree_extra_bits[i], storage_ix, storage);
        break;
      case BROTLI_REPEAT_ZERO_CODE_LENGTH:
        BrotliWriteBits(3, huffman_tree_extra_bits[i], storage_ix, storage);
        break;
    }
  }
}


static BROTLI_INLINE uint32_t BitMask(uint32_t n) {
  if (BROTLI_IS_CONSTANT(n) || BROTLI_HAS_UBFX) {
    /* Masking with this expression turns to a single
       "Unsigned Bit Field Extract" UBFX instruction on ARM. */
    return ~((0xFFFFFFFFu) << n);
  } else {
    return kBrotliBitMask[n];
  }
}


static BROTLI_INLINE void TakeDistanceFromRingBuffer(BrotliDecoderState* s) {
  int offset = s->distance_code - 3;
  if (s->distance_code <= 3) {
    /* Compensate double distance-ring-buffer roll for dictionary items. */
    s->distance_context = 1 >> s->distance_code;
    s->distance_code = s->dist_rb[(s->dist_rb_idx - offset) & 3];
    s->dist_rb_idx -= s->distance_context;
  } else {
    int index_delta = 3;
    int delta;
    int base = s->distance_code - 10;
    if (s->distance_code < 10) {
      base = s->distance_code - 4;
    } else {
      index_delta = 2;
    }
    /* Unpack one of six 4-bit values. */
    delta = ((0x605142 >> (4 * base)) & 0xF) - 3;
    s->distance_code = s->dist_rb[(s->dist_rb_idx + index_delta) & 0x3] + delta;
    if (s->distance_code <= 0) {
      /* A huge distance will cause a BROTLI_FAILURE() soon.
         This is a little faster than failing here. */
      s->distance_code = 0x7FFFFFFF;
    }
  }
}


static void StoreVarLenUint8(size_t n, size_t* storage_ix, uint8_t* storage) {
  if (n == 0) {
    BrotliWriteBits(1, 0, storage_ix, storage);
  } else {
    size_t nbits = Log2FloorNonZero(n);
    BrotliWriteBits(1, 1, storage_ix, storage);
    BrotliWriteBits(3, nbits, storage_ix, storage);
    BrotliWriteBits(nbits, n - ((size_t)1 << nbits), storage_ix, storage);
  }
}


static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
  if (s->should_wrap_ringbuffer) {
    memcpy(s->ringbuffer, s->ringbuffer_end, (size_t)s->pos);
    s->should_wrap_ringbuffer = 0;
  }
}


static void CalculateDistanceLut(BrotliDecoderState* s) {
  BrotliMetablockBodyArena* b = &s->arena.body;
  uint32_t npostfix = s->distance_postfix_bits;
  uint32_t ndirect = s->num_direct_distance_codes;
  uint32_t alphabet_size_limit = s->distance_hgroup.alphabet_size_limit;
  uint32_t postfix = 1u << npostfix;
  uint32_t j;
  uint32_t bits = 1;
  uint32_t half = 0;

  /* Skip short codes. */
  uint32_t i = BROTLI_NUM_DISTANCE_SHORT_CODES;

  /* Fill direct codes. */
  for (j = 0; j < ndirect; ++j) {
    b->dist_extra_bits[i] = 0;
    b->dist_offset[i] = j + 1;
    ++i;
  }

  /* Fill regular distance codes. */
  while (i < alphabet_size_limit) {
    uint32_t base = ndirect + ((((2 + half) << bits) - 4) << npostfix) + 1;
    /* Always fill the complete group. */
    for (j = 0; j < postfix; ++j) {
      b->dist_extra_bits[i] = (uint8_t)bits;
      b->dist_offset[i] = base + j;
      ++i;
    }
    bits = bits + half;
    half = half ^ 1;
  }
}


static BROTLI_NOINLINE BrotliDecoderErrorCode ProcessCommands(
    BrotliDecoderState* s) {
  return ProcessCommandsInternal(0, s);
}


static BrotliDecoderErrorCode ReadCodeLengthCodeLengths(BrotliDecoderState* s) {
  BrotliBitReader* br = &s->br;
  BrotliMetablockHeaderArena* h = &s->arena.header;
  uint32_t num_codes = h->repeat;
  unsigned space = h->space;
  uint32_t i = h->sub_loop_counter;
  for (; i < BROTLI_CODE_LENGTH_CODES; ++i) {
    const uint8_t code_len_idx = kCodeLengthCodeOrder[i];
    uint32_t ix;
    uint32_t v;
    if (BROTLI_PREDICT_FALSE(!BrotliSafeGetBits(br, 4, &ix))) {
      uint32_t available_bits = BrotliGetAvailableBits(br);
      if (available_bits != 0) {
        ix = BrotliGetBitsUnmasked(br) & 0xF;
      } else {
        ix = 0;
      }
      if (kCodeLengthPrefixLength[ix] > available_bits) {
        h->sub_loop_counter = i;
        h->repeat = num_codes;
        h->space = space;
        h->substate_huffman = BROTLI_STATE_HUFFMAN_COMPLEX;
        return BROTLI_DECODER_NEEDS_MORE_INPUT;
      }
    }
    v = kCodeLengthPrefixValue[ix];
    BrotliDropBits(br, kCodeLengthPrefixLength[ix]);
    h->code_length_code_lengths[code_len_idx] = (uint8_t)v;
    BROTLI_LOG_ARRAY_INDEX(h->code_length_code_lengths, code_len_idx);
    if (v != 0) {
      space = space - (32U >> v);
      ++num_codes;
      ++h->code_length_histo[v];
      if (space - 1U >= 32U) {
        /* space is 0 or wrapped around. */
        break;
      }
    }
  }
  if (!(num_codes == 1 || space == 0)) {
    return BROTLI_FAILURE(BROTLI_DECODER_ERROR_FORMAT_CL_SPACE);
  }
  return BROTLI_DECODER_SUCCESS;
}


static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
  uint64_t value = BROTLI_BSWAP64_(v);
  BrotliUnalignedWrite64(p, value);
}


int main(int argc, char** argv) {
  Context ctx;
  BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
  size_t available_in;
  const uint8_t* next_in;
  size_t available_out = BUFFER_SIZE;
  uint8_t* next_out;
  init(&ctx);

  ctx.fin = fdopen(STDIN_FILENO, "rb");
  if (!ctx.fin) fail(&ctx, "can't open input file");
  ctx.fout = fdopen(STDOUT_FILENO, "wb");
  if (!ctx.fout) fail(&ctx, "can't open output file");
  ctx.input_buffer = (uint8_t*)malloc(BUFFER_SIZE);
  if (!ctx.input_buffer) fail(&ctx, "out of memory / input buffer");
  ctx.output_buffer = (uint8_t*)malloc(BUFFER_SIZE);
  if (!ctx.output_buffer) fail(&ctx, "out of memory / output buffer");
  ctx.decoder = BrotliDecoderCreateInstance(0, 0, 0);
  if (!ctx.decoder) fail(&ctx, "out of memory / decoder");
  BrotliDecoderSetParameter(ctx.decoder, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);

  next_out = ctx.output_buffer;
  while (1) {
    if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
      if (feof(ctx.fin)) break;
      available_in = fread(ctx.input_buffer, 1, BUFFER_SIZE, ctx.fin);
      next_in = ctx.input_buffer;
      if (ferror(ctx.fin)) break;
    } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
      fwrite(ctx.output_buffer, 1, BUFFER_SIZE, ctx.fout);
      if (ferror(ctx.fout)) break;
      available_out = BUFFER_SIZE;
      next_out = ctx.output_buffer;
    } else {
      break;
    }
    result = BrotliDecoderDecompressStream(
        ctx.decoder, &available_in, &next_in, &available_out, &next_out, 0);
  }
  if (next_out != ctx.output_buffer) {
    fwrite(ctx.output_buffer, 1, next_out - ctx.output_buffer, ctx.fout);
  }
  if ((result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) || ferror(ctx.fout)) {
    fail(&ctx, "failed to write output");
  } else if (result != BROTLI_DECODER_RESULT_SUCCESS) {
    fail(&ctx, "corrupt input");
  }
  cleanup(&ctx);
  return 0;
}


static BROTLI_INLINE void PreloadSymbol(int safe,
                                        const HuffmanCode* table,
                                        BrotliBitReader* br,
                                        uint32_t* bits,
                                        uint32_t* value) {
  if (safe) {
    return;
  }
  BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
  BROTLI_HC_ADJUST_TABLE_INDEX(table, BrotliGetBits(br, HUFFMAN_TABLE_BITS));
  *bits = BROTLI_HC_FAST_LOAD_BITS(table);
  *value = BROTLI_HC_FAST_LOAD_VALUE(table);
}


void BrotliDecoderDestroyInstance(BrotliDecoderState* state) {
  if (!state) {
    return;
  } else {
    brotli_free_func free_func = state->free_func;
    void* opaque = state->memory_manager_opaque;
    BrotliDecoderStateCleanup(state);
    free_func(opaque, state);
  }
}


static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
  return *(const uint16_t*)p;
}


void ConvertToPixels(int** histo, uint8_t** pixel) {
  int height = FLAGS_height;
  int width = FLAGS_width;

  int maxs = 0;
  for (int i = 0; i < height; i++) {
    for (int j = 0; j < width; j++) {
      if (maxs < histo[i][j]) maxs = histo[i][j];
    }
  }

  bool simple = FLAGS_simple;
  double max_histo = static_cast<double>(maxs);
  for (int i = 0; i < height; i++) {
    for (int j = 0; j < width; j++) {
      if (simple) {
        pixel[i][j] = histo[i][j] > 0 ? 0 : 255;
      } else {
        pixel[i][j] = static_cast<uint8_t>(
            255 - DensityTransform(histo[i][j] / max_histo * 255));
      }
    }
  }
}


static BrotliDecoderErrorCode ReadSimpleHuffmanSymbols(
    uint32_t alphabet_size_max, uint32_t alphabet_size_limit,
    BrotliDecoderState* s) {
  /* max_bits == 1..11; symbol == 0..3; 1..44 bits will be read. */
  BrotliBitReader* br = &s->br;
  BrotliMetablockHeaderArena* h = &s->arena.header;
  uint32_t max_bits = Log2Floor(alphabet_size_max - 1);
  uint32_t i = h->sub_loop_counter;
  uint32_t num_symbols = h->symbol;
  while (i <= num_symbols) {
    uint32_t v;
    if (BROTLI_PREDICT_FALSE(!BrotliSafeReadBits(br, max_bits, &v))) {
      h->sub_loop_counter = i;
      h->substate_huffman = BROTLI_STATE_HUFFMAN_SIMPLE_READ;
      return BROTLI_DECODER_NEEDS_MORE_INPUT;
    }
    if (v >= alphabet_size_limit) {
      return
          BROTLI_FAILURE(BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET);
    }
    h->symbols_lists_array[i] = (uint16_t)v;
    BROTLI_LOG_UINT(h->symbols_lists_array[i]);
    ++i;
  }

  for (i = 0; i < num_symbols; ++i) {
    uint32_t k = i + 1;
    for (; k <= num_symbols; ++k) {
      if (h->symbols_lists_array[i] == h->symbols_lists_array[k]) {
        return BROTLI_FAILURE(BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME);
      }
    }
  }

  return BROTLI_DECODER_SUCCESS;
}


static void InitBlockTypeCodeCalculator(BlockTypeCodeCalculator* self) {
  self->last_type = 1;
  self->second_last_type = 0;
}


static void BROTLI_NOINLINE DecodeDistanceBlockSwitch(BrotliDecoderState* s) {
  DecodeDistanceBlockSwitchInternal(0, s);
}


void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s) {
  s->meta_block_remaining_len = 0;
  s->block_length[0] = 1U << 24;
  s->block_length[1] = 1U << 24;
  s->block_length[2] = 1U << 24;
  s->num_block_types[0] = 1;
  s->num_block_types[1] = 1;
  s->num_block_types[2] = 1;
  s->block_type_rb[0] = 1;
  s->block_type_rb[1] = 0;
  s->block_type_rb[2] = 1;
  s->block_type_rb[3] = 0;
  s->block_type_rb[4] = 1;
  s->block_type_rb[5] = 0;
  s->context_map = NULL;
  s->context_modes = NULL;
  s->dist_context_map = NULL;
  s->context_map_slice = NULL;
  s->literal_htree = NULL;
  s->dist_context_map_slice = NULL;
  s->dist_htree_index = 0;
  s->context_lookup = NULL;
  s->literal_hgroup.codes = NULL;
  s->literal_hgroup.htrees = NULL;
  s->insert_copy_hgroup.codes = NULL;
  s->insert_copy_hgroup.htrees = NULL;
  s->distance_hgroup.codes = NULL;
  s->distance_hgroup.htrees = NULL;
}


static void ppp_tx_cp(struct net_device *dev, u16 pid, u8 code,
		      u8 id, unsigned int len, const void *data)
{
	struct sk_buff *skb;
	struct cp_header *cp;
	unsigned int magic_len = 0;
	static u32 magic;

#if DEBUG_CP
	int i;
	char *ptr;
#endif

	if (pid == PID_LCP && (code == LCP_ECHO_REQ || code == LCP_ECHO_REPLY))
		magic_len = sizeof(magic);

	skb = dev_alloc_skb(sizeof(struct hdlc_header) +
			    sizeof(struct cp_header) + magic_len + len);
	if (!skb) {
		netdev_warn(dev, "out of memory in ppp_tx_cp()\n");
		return;
	}
	skb_reserve(skb, sizeof(struct hdlc_header));

	cp = skb_put(skb, sizeof(struct cp_header));
	cp->code = code;
	cp->id = id;
	cp->len = htons(sizeof(struct cp_header) + magic_len + len);

	if (magic_len)
		skb_put_data(skb, &magic, magic_len);
	if (len)
		skb_put_data(skb, data, len);

#if DEBUG_CP
	BUG_ON(code >= CP_CODES);
	ptr = debug_buffer;
	*ptr = '\x0';
	for (i = 0; i < min_t(unsigned int, magic_len + len, DEBUG_CP); i++) {
		sprintf(ptr, " %02X", skb->data[sizeof(struct cp_header) + i]);
		ptr += strlen(ptr);
	}
	printk(KERN_DEBUG "%s: TX %s [%s id 0x%X]%s\n", dev->name,
	       proto_name(pid), code_names[code], id, debug_buffer);
#endif

	ppp_hard_header(skb, dev, pid, NULL, NULL, 0);

	skb->priority = TC_PRIO_CONTROL;
	skb->dev = dev;
	skb_reset_network_header(skb);
	skb_queue_tail(&tx_queue, skb);
}


static int hugetlb_acct_memory(struct hstate *h, long delta)
{
	int ret = -ENOMEM;

	spin_lock(&hugetlb_lock);
	/*
	 * When cpuset is configured, it breaks the strict hugetlb page
	 * reservation as the accounting is done on a global variable. Such
	 * reservation is completely rubbish in the presence of cpuset because
	 * the reservation is not checked against page availability for the
	 * current cpuset. Application can still potentially OOM'ed by kernel
	 * with lack of free htlb page in cpuset that the task is in.
	 * Attempt to enforce strict accounting with cpuset is almost
	 * impossible (or too ugly) because cpuset is too fluid that
	 * task or memory node can be dynamically moved between cpusets.
	 *
	 * The change of semantics for shared hugetlb mapping with cpuset is
	 * undesirable. However, in order to preserve some of the semantics,
	 * we fall back to check against current free page availability as
	 * a best attempt and hopefully to minimize the impact of changing
	 * semantics that cpuset has.
	 *
	 * Apart from cpuset, we also have memory policy mechanism that
	 * also determines from which node the kernel will allocate memory
	 * in a NUMA system. So similar to cpuset, we also should consider
	 * the memory policy of the current task. Similar to the description
	 * above.
	 */
	if (delta > 0) {
		if (gather_surplus_pages(h, delta) < 0)
			goto out;

		if (delta > allowed_mems_nr(h)) {
			return_unused_surplus_pages(h, delta);
			goto out;
		}
	}

	ret = 0;
	if (delta < 0)
		return_unused_surplus_pages(h, (unsigned long) -delta);

out:
	spin_unlock(&hugetlb_lock);
	return ret;
}


resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
					  struct hugetlb_cgroup *h_cg,
					  struct hstate *h)
{
#ifdef CONFIG_CGROUP_HUGETLB
	if (!h_cg || !h) {
		resv_map->reservation_counter = NULL;
		resv_map->pages_per_hpage = 0;
		resv_map->css = NULL;
	} else {
		resv_map->reservation_counter =
			&h_cg->rsvd_hugepage[hstate_index(h)];
		resv_map->pages_per_hpage = pages_per_huge_page(h);
		resv_map->css = &h_cg->css;
	}
#endif
}


pte_t *huge_pte_offset(struct mm_struct *mm,
		       unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (!pgd_present(*pgd))
		return NULL;
	p4d = p4d_offset(pgd, addr);
	if (!p4d_present(*p4d))
		return NULL;

	pud = pud_offset(p4d, addr);
	if (sz == PUD_SIZE)
		/* must be pud huge, non-present or none */
		return (pte_t *)pud;
	if (!pud_present(*pud))
		return NULL;
	/* must have a valid entry and size to go further */

	pmd = pmd_offset(pud, addr);
	/* must be pmd huge, non-present or none */
	return (pte_t *)pmd;
}


bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
{
	return size == HPAGE_SIZE;
}


static bool rbd_obj_advance_read(struct rbd_obj_request *obj_req, int *result)
{
	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
	int ret;

again:
	switch (obj_req->read_state) {
	case RBD_OBJ_READ_START:
		rbd_assert(!*result);

		if (!rbd_obj_may_exist(obj_req)) {
			*result = -ENOENT;
			obj_req->read_state = RBD_OBJ_READ_OBJECT;
			goto again;
		}

		ret = rbd_obj_read_object(obj_req);
		if (ret) {
			*result = ret;
			return true;
		}
		obj_req->read_state = RBD_OBJ_READ_OBJECT;
		return false;
	case RBD_OBJ_READ_OBJECT:
		if (*result == -ENOENT && rbd_dev->parent_overlap) {
			/* reverse map this object extent onto the parent */
			ret = rbd_obj_calc_img_extents(obj_req, false);
			if (ret) {
				*result = ret;
				return true;
			}
			if (obj_req->num_img_extents) {
				ret = rbd_obj_read_from_parent(obj_req);
				if (ret) {
					*result = ret;
					return true;
				}
				obj_req->read_state = RBD_OBJ_READ_PARENT;
				return false;
			}
		}

		/*
		 * -ENOENT means a hole in the image -- zero-fill the entire
		 * length of the request.  A short read also implies zero-fill
		 * to the end of the request.
		 */
		if (*result == -ENOENT) {
			rbd_obj_zero_range(obj_req, 0, obj_req->ex.oe_len);
			*result = 0;
		} else if (*result >= 0) {
			if (*result < obj_req->ex.oe_len)
				rbd_obj_zero_range(obj_req, *result,
						obj_req->ex.oe_len - *result);
			else
				rbd_assert(*result == obj_req->ex.oe_len);
			*result = 0;
		}
		return true;
	case RBD_OBJ_READ_PARENT:
		/*
		 * The parent image is read only up to the overlap -- zero-fill
		 * from the overlap to the end of the request.
		 */
		if (!*result) {
			u32 obj_overlap = rbd_obj_img_extents_bytes(obj_req);

			if (obj_overlap < obj_req->ex.oe_len)
				rbd_obj_zero_range(obj_req, obj_overlap,
					    obj_req->ex.oe_len - obj_overlap);
		}
		return true;
	default:
		BUG();
	}
}


static void rbd_dev_parent_put(struct rbd_device *rbd_dev)
{
	int counter;

	if (!rbd_dev->parent_spec)
		return;

	counter = atomic_dec_return_safe(&rbd_dev->parent_ref);
	if (counter > 0)
		return;

	/* Last reference; clean up parent data structures */

	if (!counter)
		rbd_dev_unparent(rbd_dev);
	else
		rbd_warn(rbd_dev, "parent reference underflow");
}


static u64 rbd_obj_img_extents_bytes(struct rbd_obj_request *obj_req)
{
	return ceph_file_extents_bytes(obj_req->img_extents,
				       obj_req->num_img_extents);
}


static int rbd_osd_setup_stat(struct ceph_osd_request *osd_req, int which)
{
	struct page **pages;

	/*
	 * The response data for a STAT call consists of:
	 *     le64 length;
	 *     struct {
	 *         le32 tv_sec;
	 *         le32 tv_nsec;
	 *     } mtime;
	 */
	pages = ceph_alloc_page_vector(1, GFP_NOIO);
	if (IS_ERR(pages))
		return PTR_ERR(pages);

	osd_req_op_init(osd_req, which, CEPH_OSD_OP_STAT, 0);
	osd_req_op_raw_data_in_pages(osd_req, which, pages,
				     8 + sizeof(struct ceph_timespec),
				     0, false, true);
	return 0;
}


static void rbd_img_object_requests(struct rbd_img_request *img_req)
{
	struct rbd_obj_request *obj_req;

	rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);

	for_each_obj_request(img_req, obj_req) {
		int result = 0;

		if (__rbd_obj_handle_request(obj_req, &result)) {
			if (result) {
				img_req->pending.result = result;
				return;
			}
		} else {
			img_req->pending.num_pending++;
		}
	}
}


static int rbd_add_acquire_lock(struct rbd_device *rbd_dev)
{
	long ret;

	if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK)) {
		if (!rbd_dev->opts->exclusive && !rbd_dev->opts->lock_on_read)
			return 0;

		rbd_warn(rbd_dev, "exclusive-lock feature is not enabled");
		return -EINVAL;
	}

	if (rbd_is_ro(rbd_dev))
		return 0;

	rbd_assert(!rbd_is_lock_owner(rbd_dev));
	queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
	ret = wait_for_completion_killable_timeout(&rbd_dev->acquire_wait,
			    ceph_timeout_jiffies(rbd_dev->opts->lock_timeout));
	if (ret > 0) {
		ret = rbd_dev->acquire_err;
	} else {
		cancel_delayed_work_sync(&rbd_dev->lock_dwork);
		if (!ret)
			ret = -ETIMEDOUT;
	}

	if (ret) {
		rbd_warn(rbd_dev, "failed to acquire exclusive lock: %ld", ret);
		return ret;
	}

	/*
	 * The lock may have been released by now, unless automatic lock
	 * transitions are disabled.
	 */
	rbd_assert(!rbd_dev->opts->exclusive || rbd_is_lock_owner(rbd_dev));
	return 0;
}


static void rbd_dev_device_release(struct rbd_device *rbd_dev)
{
	clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
	rbd_free_disk(rbd_dev);
	if (!single_major)
		unregister_blkdev(rbd_dev->major, rbd_dev->name);
}


static int rbd_dev_image_id(struct rbd_device *rbd_dev)
{
	int ret;
	size_t size;
	CEPH_DEFINE_OID_ONSTACK(oid);
	void *response;
	char *image_id;

	/*
	 * When probing a parent image, the image id is already
	 * known (and the image name likely is not).  There's no
	 * need to fetch the image id again in this case.  We
	 * do still need to set the image format though.
	 */
	if (rbd_dev->spec->image_id) {
		rbd_dev->image_format = *rbd_dev->spec->image_id ? 2 : 1;

		return 0;
	}

	/*
	 * First, see if the format 2 image id file exists, and if
	 * so, get the image's persistent id from it.
	 */
	ret = ceph_oid_aprintf(&oid, GFP_KERNEL, "%s%s", RBD_ID_PREFIX,
			       rbd_dev->spec->image_name);
	if (ret)
		return ret;

	dout("rbd id object name is %s\n", oid.name);

	/* Response will be an encoded string, which includes a length */
	size = sizeof (__le32) + RBD_IMAGE_ID_LEN_MAX;
	response = kzalloc(size, GFP_NOIO);
	if (!response) {
		ret = -ENOMEM;
		goto out;
	}

	/* If it doesn't exist we'll assume it's a format 1 image */

	ret = rbd_obj_method_sync(rbd_dev, &oid, &rbd_dev->header_oloc,
				  "get_id", NULL, 0,
				  response, size);
	dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
	if (ret == -ENOENT) {
		image_id = kstrdup("", GFP_KERNEL);
		ret = image_id ? 0 : -ENOMEM;
		if (!ret)
			rbd_dev->image_format = 1;
	} else if (ret >= 0) {
		void *p = response;

		image_id = ceph_extract_encoded_string(&p, p + ret,
						NULL, GFP_NOIO);
		ret = PTR_ERR_OR_ZERO(image_id);
		if (!ret)
			rbd_dev->image_format = 2;
	}

	if (!ret) {
		rbd_dev->spec->image_id = image_id;
		dout("image_id is %s\n", image_id);
	}
out:
	kfree(response);
	ceph_oid_destroy(&oid);
	return ret;
}


static void zero_bvec(struct bio_vec *bv)
{
	void *buf;
	unsigned long flags;

	buf = bvec_kmap_irq(bv, &flags);
	memset(buf, 0, bv->bv_len);
	flush_dcache_page(bv->bv_page);
	bvec_kunmap_irq(buf, &flags);
}