static int io_send(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	struct iovec iov;
	struct socket *sock;
	unsigned flags;
	int ret;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
	if (unlikely(ret))
		return ret;

	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;

	flags = req->sr_msg.msg_flags;
	if (flags & MSG_DONTWAIT)
		req->flags |= REQ_F_NOWAIT;
	else if (issue_flags & IO_URING_F_NONBLOCK)
		flags |= MSG_DONTWAIT;

	msg.msg_flags = flags;
	ret = sock_sendmsg(sock, &msg);
	if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;

	if (ret < 0)
		req_set_fail_links(req);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *creds;

	creds = idr_remove(&ctx->personality_idr, id);
	if (creds) {
		put_cred(creds);
		return 0;
	}

	return -EINVAL;


static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	if (ctx->cq_ev_fd) {
		eventfd_ctx_put(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return 0;
	}

	return -ENXIO;


 */
static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
{
	struct file *file;
#if defined(CONFIG_UNIX)
	int ret;

	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ERR_PTR(ret);
#endif

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
#if defined(CONFIG_UNIX)
	if (IS_ERR(file)) {
		sock_release(ctx->ring_sock);
		ctx->ring_sock = NULL;
	} else {
		ctx->ring_sock->file = file;
	}
#endif
	return file;


static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
{
	struct io_submit_state *state = &ctx->submit_state;
	struct io_comp_state *cs = &state->comp;
	struct io_kiocb *req = NULL;

	/*
	 * If we have more than a batch's worth of requests in our IRQ side
	 * locked cache, grab the lock and move them over to our submission
	 * side cache.
	 */
	if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH) {
		spin_lock_irq(&ctx->completion_lock);
		list_splice_init(&cs->locked_free_list, &cs->free_list);
		cs->locked_free_nr = 0;
		spin_unlock_irq(&ctx->completion_lock);
	}

	while (!list_empty(&cs->free_list)) {
		req = list_first_entry(&cs->free_list, struct io_kiocb,
					compl.list);
		list_del(&req->compl.list);
		state->reqs[state->free_reqs++] = req;
		if (state->free_reqs == ARRAY_SIZE(state->reqs))
			break;
	}

	return req != NULL;
}


static void io_submit_flush_completions(struct io_comp_state *cs,
					struct io_ring_ctx *ctx)
{
	int i, nr = cs->nr;
	struct io_kiocb *req;
	struct req_batch rb;

	io_init_req_batch(&rb);
	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < nr; i++) {
		req = cs->reqs[i];
		__io_cqring_fill_event(req, req->result, req->compl.cflags);
	}
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	for (i = 0; i < nr; i++) {
		req = cs->reqs[i];

		/* submission and completion refs */
		if (refcount_sub_and_test(2, &req->refs))
			io_req_free_batch(&rb, req, &ctx->submit_state);
	}

	io_req_free_batch_finish(ctx, &rb);
	cs->nr = 0;
}


static __poll_t io_uring_poll(struct file *file, poll_table *wait)
{
	struct io_ring_ctx *ctx = file->private_data;
	__poll_t mask = 0;

	poll_wait(file, &ctx->cq_wait, wait);
	/*
	 * synchronizes with barrier from wq_has_sleeper call in
	 * io_commit_cqring
	 */
	smp_rmb();
	if (!io_sqring_full(ctx))
		mask |= EPOLLOUT | EPOLLWRNORM;

	/*
	 * Don't flush cqring overflow list here, just do a simple check.
	 * Otherwise there could possible be ABBA deadlock:
	 *      CPU0                    CPU1
	 *      ----                    ----
	 * lock(&ctx->uring_lock);
	 *                              lock(&ep->mtx);
	 *                              lock(&ctx->uring_lock);
	 * lock(&ep->mtx);
	 *
	 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
	 * pushs them to do the flush.
	 */
	if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow))
		mask |= EPOLLIN | EPOLLRDNORM;

	return mask;


static bool io_sq_thread_should_stop(struct io_sq_data *sqd)
{
	return test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);


static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
			   struct iov_iter *iter, bool needs_lock)
{
	void __user *buf = u64_to_user_ptr(req->rw.addr);
	size_t sqe_len = req->rw.len;
	u8 opcode = req->opcode;
	ssize_t ret;

	if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
		*iovec = NULL;
		return io_import_fixed(req, rw, iter);
	}

	/* buffer index only valid with fixed read/write, or buffer select  */
	if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
		return -EINVAL;

	if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
		if (req->flags & REQ_F_BUFFER_SELECT) {
			buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
			if (IS_ERR(buf))
				return PTR_ERR(buf);
			req->rw.len = sqe_len;
		}

		ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
		*iovec = NULL;
		return ret;
	}

	if (req->flags & REQ_F_BUFFER_SELECT) {
		ret = io_iov_buffer_select(req, *iovec, needs_lock);
		if (!ret)
			iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
		*iovec = NULL;
		return ret;
	}

	return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
			      req->ctx->compat);
}


static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
				    unsigned int nr_args)
{
	struct io_uring_restriction *res;
	size_t size;
	int i, ret;

	/* Restrictions allowed only if rings started disabled */
	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
		return -EBADFD;

	/* We allow only a single restrictions registration */
	if (ctx->restrictions.registered)
		return -EBUSY;

	if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
		return -EINVAL;

	size = array_size(nr_args, sizeof(*res));
	if (size == SIZE_MAX)
		return -EOVERFLOW;

	res = memdup_user(arg, size);
	if (IS_ERR(res))
		return PTR_ERR(res);

	ret = 0;

	for (i = 0; i < nr_args; i++) {
		switch (res[i].opcode) {
		case IORING_RESTRICTION_REGISTER_OP:
			if (res[i].register_op >= IORING_REGISTER_LAST) {
				ret = -EINVAL;
				goto out;
			}

			__set_bit(res[i].register_op,
				  ctx->restrictions.register_op);
			break;
		case IORING_RESTRICTION_SQE_OP:
			if (res[i].sqe_op >= IORING_OP_LAST) {
				ret = -EINVAL;
				goto out;
			}

			__set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
			break;
		case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
			ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
			break;
		case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
			ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
			break;
		default:
			ret = -EINVAL;
			goto out;
		}
	}

out:
	/* Reset all restrictions if an error happened */
	if (ret != 0)
		memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
	else
		ctx->restrictions.registered = true;

	kfree(res);
	return ret;


static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

	__io_complete_rw(req, res, res2, 0);
}


static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
{
	unsigned int nr_events = 0;
	int iters = 0, ret = 0;

	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
	do {
		/*
		 * Don't enter poll loop if we already have events pending.
		 * If we do, we can potentially be spinning for commands that
		 * already triggered a CQE (eg in error).
		 */
		if (test_bit(0, &ctx->cq_check_overflow))
			__io_cqring_overflow_flush(ctx, false, NULL, NULL);
		if (io_cqring_events(ctx))
			break;

		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (!(++iters & 7)) {
			mutex_unlock(&ctx->uring_lock);
			io_run_task_work();
			mutex_lock(&ctx->uring_lock);
		}

		ret = io_iopoll_getevents(ctx, &nr_events, min);
		if (ret <= 0)
			break;
		ret = 0;
	} while (min && !nr_events && !need_resched());

	mutex_unlock(&ctx->uring_lock);
	return ret;
}


static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node)
{
	struct fixed_rsrc_data *rsrc_data = ref_node->rsrc_data;
	struct io_ring_ctx *ctx = rsrc_data->ctx;
	struct io_rsrc_put *prsrc, *tmp;

	list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
		list_del(&prsrc->list);
		ref_node->rsrc_put(ctx, prsrc);
		kfree(prsrc);
	}

	percpu_ref_exit(&ref_node->refs);
	kfree(ref_node);
	percpu_ref_put(&rsrc_data->refs);


static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
				     struct req_batch *rb)
{
	if (rb->task)
		io_put_task(rb->task, rb->task_refs);
	if (rb->ctx_refs)
		percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
}


static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);
		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;


static inline u64 intel_get_tsx_transaction(u64 tsx_tuning, u64 ax)
{
	u64 txn = (tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;

	/* For RTM XABORTs also log the abort code from AX */
	if ((txn & PERF_TXN_TRANSACTION) && (ax & 1))
		txn |= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
	return txn;
}


static inline u64 get_pebs_status(void *n)
{
	if (x86_pmu.intel_cap.pebs_format < 4)
		return ((struct pebs_record_nhm *)n)->status;
	return ((struct pebs_basic *)n)->applicable_counters;
}


void intel_pmu_pebs_enable_all(void)
{
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	if (cpuc->pebs_enabled)
		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
}


static void adaptive_pebs_record_size_update(void)
{
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	u64 pebs_data_cfg = cpuc->pebs_data_cfg;
	int sz = sizeof(struct pebs_basic);

	if (pebs_data_cfg & PEBS_DATACFG_MEMINFO)
		sz += sizeof(struct pebs_meminfo);
	if (pebs_data_cfg & PEBS_DATACFG_GP)
		sz += sizeof(struct pebs_gprs);
	if (pebs_data_cfg & PEBS_DATACFG_XMMS)
		sz += sizeof(struct pebs_xmm);
	if (pebs_data_cfg & PEBS_DATACFG_LBRS)
		sz += x86_pmu.lbr_nr * sizeof(struct lbr_entry);

	cpuc->pebs_record_size = sz;
}


static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
{
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	struct debug_store *ds = cpuc->ds;
	struct perf_event *event = cpuc->events[0]; /* PMC0 only */
	struct pebs_record_core *at, *top;
	int n;

	if (!x86_pmu.pebs_active)
		return;

	at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;

	/*
	 * Whatever else happens, drain the thing
	 */
	ds->pebs_index = ds->pebs_buffer_base;

	if (!test_bit(0, cpuc->active_mask))
		return;

	WARN_ON_ONCE(!event);

	if (!event->attr.precise_ip)
		return;

	n = top - at;
	if (n <= 0) {
		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
			intel_pmu_save_and_restart_reload(event, 0);
		return;
	}

	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,
			       setup_pebs_fixed_sample_data);
}


void init_debug_store_on_cpu(int cpu)
{
	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	if (!ds)
		return;

	wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
		     (u32)((u64)(unsigned long)ds),
		     (u32)((u64)(unsigned long)ds >> 32));
}


pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc,
		  struct perf_event *event, bool add)
{
	struct pmu *pmu = event->ctx->pmu;
	/*
	 * Make sure we get updated with the first PEBS
	 * event. It will trigger also during removal, but
	 * that does not hurt:
	 */
	bool update = cpuc->n_pebs == 1;

	if (needed_cb != pebs_needs_sched_cb(cpuc)) {
		if (!needed_cb)
			perf_sched_cb_inc(pmu);
		else
			perf_sched_cb_dec(pmu);

		update = true;
	}

	/*
	 * The PEBS record doesn't shrink on pmu::del(). Doing so would require
	 * iterating all remaining PEBS events to reconstruct the config.
	 */
	if (x86_pmu.intel_cap.pebs_baseline && add) {
		u64 pebs_data_cfg;

		/* Clear pebs_data_cfg and pebs_record_size for first PEBS. */
		if (cpuc->n_pebs == 1) {
			cpuc->pebs_data_cfg = 0;
			cpuc->pebs_record_size = sizeof(struct pebs_basic);
		}

		pebs_data_cfg = pebs_update_adaptive_cfg(event);

		/* Update pebs_record_size if new event requires more data. */
		if (pebs_data_cfg & ~cpuc->pebs_data_cfg) {
			cpuc->pebs_data_cfg |= pebs_data_cfg;
			adaptive_pebs_record_size_update();
			update = true;
		}
	}

	if (update)
		pebs_update_threshold(cpuc);
}


int btrfs_search_slot_for_read(struct btrfs_root *root,
			       const struct btrfs_key *key,
			       struct btrfs_path *p, int find_higher,
			       int return_any)
{
	int ret;
	struct extent_buffer *leaf;

again:
	ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
	if (ret <= 0)
		return ret;
	/*
	 * a return value of 1 means the path is at the position where the
	 * item should be inserted. Normally this is the next bigger item,
	 * but in case the previous item is the last in a leaf, path points
	 * to the first free slot in the previous leaf, i.e. at an invalid
	 * item.
	 */
	leaf = p->nodes[0];

	if (find_higher) {
		if (p->slots[0] >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, p);
			if (ret <= 0)
				return ret;
			if (!return_any)
				return 1;
			/*
			 * no higher item found, return the next
			 * lower instead
			 */
			return_any = 0;
			find_higher = 0;
			btrfs_release_path(p);
			goto again;
		}
	} else {
		if (p->slots[0] == 0) {
			ret = btrfs_prev_leaf(root, p);
			if (ret < 0)
				return ret;
			if (!ret) {
				leaf = p->nodes[0];
				if (p->slots[0] == btrfs_header_nritems(leaf))
					p->slots[0]--;
				return 0;
			}
			if (!return_any)
				return 1;
			/*
			 * no lower item found, return the next
			 * higher instead
			 */
			return_any = 0;
			find_higher = 1;
			btrfs_release_path(p);
			goto again;
		} else {
			--p->slots[0];
		}
	}
	return 0;
}


static noinline int __push_leaf_right(struct btrfs_path *path,
				      int data_size, int empty,
				      struct extent_buffer *right,
				      int free_space, u32 left_nritems,
				      u32 min_slot)
{
	struct btrfs_fs_info *fs_info = right->fs_info;
	struct extent_buffer *left = path->nodes[0];
	struct extent_buffer *upper = path->nodes[1];
	struct btrfs_map_token token;
	struct btrfs_disk_key disk_key;
	int slot;
	u32 i;
	int push_space = 0;
	int push_items = 0;
	struct btrfs_item *item;
	u32 nr;
	u32 right_nritems;
	u32 data_end;
	u32 this_item_size;

	if (empty)
		nr = 0;
	else
		nr = max_t(u32, 1, min_slot);

	if (path->slots[0] >= left_nritems)
		push_space += data_size;

	slot = path->slots[1];
	i = left_nritems - 1;
	while (i >= nr) {
		item = btrfs_item_nr(i);

		if (!empty && push_items > 0) {
			if (path->slots[0] > i)
				break;
			if (path->slots[0] == i) {
				int space = btrfs_leaf_free_space(left);

				if (space + push_space * 2 > free_space)
					break;
			}
		}

		if (path->slots[0] == i)
			push_space += data_size;

		this_item_size = btrfs_item_size(left, item);
		if (this_item_size + sizeof(*item) + push_space > free_space)
			break;

		push_items++;
		push_space += this_item_size + sizeof(*item);
		if (i == 0)
			break;
		i--;
	}

	if (push_items == 0)
		goto out_unlock;

	WARN_ON(!empty && push_items == left_nritems);

	/* push left to right */
	right_nritems = btrfs_header_nritems(right);

	push_space = btrfs_item_end_nr(left, left_nritems - push_items);
	push_space -= leaf_data_end(left);

	/* make room in the right data area */
	data_end = leaf_data_end(right);
	memmove_extent_buffer(right,
			      BTRFS_LEAF_DATA_OFFSET + data_end - push_space,
			      BTRFS_LEAF_DATA_OFFSET + data_end,
			      BTRFS_LEAF_DATA_SIZE(fs_info) - data_end);

	/* copy from the left data area */
	copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET +
		     BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
		     BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left),
		     push_space);

	memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
			      btrfs_item_nr_offset(0),
			      right_nritems * sizeof(struct btrfs_item));

	/* copy the items from left to right */
	copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
		   btrfs_item_nr_offset(left_nritems - push_items),
		   push_items * sizeof(struct btrfs_item));

	/* update the item pointers */
	btrfs_init_map_token(&token, right);
	right_nritems += push_items;
	btrfs_set_header_nritems(right, right_nritems);
	push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
	for (i = 0; i < right_nritems; i++) {
		item = btrfs_item_nr(i);
		push_space -= btrfs_token_item_size(&token, item);
		btrfs_set_token_item_offset(&token, item, push_space);
	}

	left_nritems -= push_items;
	btrfs_set_header_nritems(left, left_nritems);

	if (left_nritems)
		btrfs_mark_buffer_dirty(left);
	else
		btrfs_clean_tree_block(left);

	btrfs_mark_buffer_dirty(right);

	btrfs_item_key(right, &disk_key, 0);
	btrfs_set_node_key(upper, &disk_key, slot + 1);
	btrfs_mark_buffer_dirty(upper);

	/* then fixup the leaf pointer in the path */
	if (path->slots[0] >= left_nritems) {
		path->slots[0] -= left_nritems;
		if (btrfs_header_nritems(path->nodes[0]) == 0)
			btrfs_clean_tree_block(path->nodes[0]);
		btrfs_tree_unlock(path->nodes[0]);
		free_extent_buffer(path->nodes[0]);
		path->nodes[0] = right;
		path->slots[1] += 1;
	} else {
		btrfs_tree_unlock(right);
		free_extent_buffer(right);
	}
	return 0;

out_unlock:
	btrfs_tree_unlock(right);
	free_extent_buffer(right);
	return 1;
}


int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
			 struct btrfs_path *path,
			 u64 min_trans)
{
	struct extent_buffer *cur;
	struct btrfs_key found_key;
	int slot;
	int sret;
	u32 nritems;
	int level;
	int ret = 1;
	int keep_locks = path->keep_locks;

	path->keep_locks = 1;
again:
	cur = btrfs_read_lock_root_node(root);
	level = btrfs_header_level(cur);
	WARN_ON(path->nodes[level]);
	path->nodes[level] = cur;
	path->locks[level] = BTRFS_READ_LOCK;

	if (btrfs_header_generation(cur) < min_trans) {
		ret = 1;
		goto out;
	}
	while (1) {
		nritems = btrfs_header_nritems(cur);
		level = btrfs_header_level(cur);
		sret = btrfs_bin_search(cur, min_key, &slot);
		if (sret < 0) {
			ret = sret;
			goto out;
		}

		/* at the lowest level, we're done, setup the path and exit */
		if (level == path->lowest_level) {
			if (slot >= nritems)
				goto find_next_key;
			ret = 0;
			path->slots[level] = slot;
			btrfs_item_key_to_cpu(cur, &found_key, slot);
			goto out;
		}
		if (sret && slot > 0)
			slot--;
		/*
		 * check this node pointer against the min_trans parameters.
		 * If it is too old, skip to the next one.
		 */
		while (slot < nritems) {
			u64 gen;

			gen = btrfs_node_ptr_generation(cur, slot);
			if (gen < min_trans) {
				slot++;
				continue;
			}
			break;
		}
find_next_key:
		/*
		 * we didn't find a candidate key in this node, walk forward
		 * and find another one
		 */
		if (slot >= nritems) {
			path->slots[level] = slot;
			sret = btrfs_find_next_key(root, path, min_key, level,
						  min_trans);
			if (sret == 0) {
				btrfs_release_path(path);
				goto again;
			} else {
				goto out;
			}
		}
		/* save our key for returning back */
		btrfs_node_key_to_cpu(cur, &found_key, slot);
		path->slots[level] = slot;
		if (level == path->lowest_level) {
			ret = 0;
			goto out;
		}
		cur = btrfs_read_node_slot(cur, slot);
		if (IS_ERR(cur)) {
			ret = PTR_ERR(cur);
			goto out;
		}

		btrfs_tree_read_lock(cur);

		path->locks[level - 1] = BTRFS_READ_LOCK;
		path->nodes[level - 1] = cur;
		unlock_up(path, level, 1, 0, NULL);
	}
out:
	path->keep_locks = keep_locks;
	if (ret == 0) {
		btrfs_unlock_up_safe(path, path->lowest_level + 1);
		memcpy(min_key, &found_key, sizeof(found_key));
	}
	return ret;
}


static bool check_sibling_keys(struct extent_buffer *left,
			       struct extent_buffer *right)
{
	struct btrfs_key left_last;
	struct btrfs_key right_first;
	int level = btrfs_header_level(left);
	int nr_left = btrfs_header_nritems(left);
	int nr_right = btrfs_header_nritems(right);

	/* No key to check in one of the tree blocks */
	if (!nr_left || !nr_right)
		return false;

	if (level) {
		btrfs_node_key_to_cpu(left, &left_last, nr_left - 1);
		btrfs_node_key_to_cpu(right, &right_first, 0);
	} else {
		btrfs_item_key_to_cpu(left, &left_last, nr_left - 1);
		btrfs_item_key_to_cpu(right, &right_first, 0);
	}

	if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) {
		btrfs_crit(left->fs_info,
"bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)",
			   left_last.objectid, left_last.type,
			   left_last.offset, right_first.objectid,
			   right_first.type, right_first.offset);
		return true;
	}
	return false;
}


static void fixup_low_keys(struct btrfs_path *path,
			   struct btrfs_disk_key *key, int level)
{
	int i;
	struct extent_buffer *t;
	int ret;

	for (i = level; i < BTRFS_MAX_LEVEL; i++) {
		int tslot = path->slots[i];

		if (!path->nodes[i])
			break;
		t = path->nodes[i];
		ret = tree_mod_log_insert_key(t, tslot, MOD_LOG_KEY_REPLACE,
				GFP_ATOMIC);
		BUG_ON(ret < 0);
		btrfs_set_node_key(t, key, tslot);
		btrfs_mark_buffer_dirty(path->nodes[i]);
		if (tslot != 0)
			break;
	}
}


int btrfs_previous_extent_item(struct btrfs_root *root,
			struct btrfs_path *path, u64 min_objectid)
{
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	u32 nritems;
	int ret;

	while (1) {
		if (path->slots[0] == 0) {
			ret = btrfs_prev_leaf(root, path);
			if (ret != 0)
				return ret;
		} else {
			path->slots[0]--;
		}
		leaf = path->nodes[0];
		nritems = btrfs_header_nritems(leaf);
		if (nritems == 0)
			return 1;
		if (path->slots[0] == nritems)
			path->slots[0]--;

		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
		if (found_key.objectid < min_objectid)
			break;
		if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
		    found_key.type == BTRFS_METADATA_ITEM_KEY)
			return 0;
		if (found_key.objectid == min_objectid &&
		    found_key.type < BTRFS_EXTENT_ITEM_KEY)
			break;
	}
	return 1;
}


static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
					 struct btrfs_root *root,
					 struct btrfs_path *path, int ins_len)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_file_extent_item *fi;
	u64 extent_len = 0;
	u32 item_size;
	int ret;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
	       key.type != BTRFS_EXTENT_CSUM_KEY);

	if (btrfs_leaf_free_space(leaf) >= ins_len)
		return 0;

	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	if (key.type == BTRFS_EXTENT_DATA_KEY) {
		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
		extent_len = btrfs_file_extent_num_bytes(leaf, fi);
	}
	btrfs_release_path(path);

	path->keep_locks = 1;
	path->search_for_split = 1;
	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
	path->search_for_split = 0;
	if (ret > 0)
		ret = -EAGAIN;
	if (ret < 0)
		goto err;

	ret = -EAGAIN;
	leaf = path->nodes[0];
	/* if our item isn't there, return now */
	if (item_size != btrfs_item_size_nr(leaf, path->slots[0]))
		goto err;

	/* the leaf has  changed, it now has room.  return now */
	if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len)
		goto err;

	if (key.type == BTRFS_EXTENT_DATA_KEY) {
		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
		if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
			goto err;
	}

	ret = split_leaf(trans, root, &key, path, ins_len, 1);
	if (ret)
		goto err;

	path->keep_locks = 0;
	btrfs_unlock_up_safe(path, 1);
	return 0;
err:
	path->keep_locks = 0;
	return ret;
}


static void mbedtls_cleanup(void)
{
  (void)Curl_mbedtlsthreadlock_thread_cleanup();
}


static CURLcode load_cafile(const char *path, br_x509_trust_anchor **anchors,
                            size_t *anchors_len)
{
  struct cafile_parser ca;
  br_pem_decoder_context pc;
  br_x509_trust_anchor *ta;
  size_t ta_size;
  br_x509_trust_anchor *new_anchors;
  size_t new_anchors_len;
  br_x509_pkey *pkey;
  FILE *fp;
  unsigned char buf[BUFSIZ], *p;
  const char *name;
  size_t n, i, pushed;

  fp = fopen(path, "rb");
  if(!fp)
    return CURLE_SSL_CACERT_BADFILE;

  ca.err = CURLE_OK;
  ca.in_cert = FALSE;
  ca.anchors = NULL;
  ca.anchors_len = 0;
  br_pem_decoder_init(&pc);
  br_pem_decoder_setdest(&pc, x509_push, &ca);
  for(;;) {
    n = fread(buf, 1, sizeof(buf), fp);
    if(n == 0)
      break;
    p = buf;
    while(n) {
      pushed = br_pem_decoder_push(&pc, p, n);
      if(ca.err)
        goto fail;
      p += pushed;
      n -= pushed;

      switch(br_pem_decoder_event(&pc)) {
      case 0:
        break;
      case BR_PEM_BEGIN_OBJ:
        name = br_pem_decoder_name(&pc);
        if(strcmp(name, "CERTIFICATE") && strcmp(name, "X509 CERTIFICATE"))
          break;
        br_x509_decoder_init(&ca.xc, append_dn, &ca);
        if(ca.anchors_len == SIZE_MAX / sizeof(ca.anchors[0])) {
          ca.err = CURLE_OUT_OF_MEMORY;
          goto fail;
        }
        new_anchors_len = ca.anchors_len + 1;
        new_anchors = realloc(ca.anchors,
                              new_anchors_len * sizeof(ca.anchors[0]));
        if(!new_anchors) {
          ca.err = CURLE_OUT_OF_MEMORY;
          goto fail;
        }
        ca.anchors = new_anchors;
        ca.anchors_len = new_anchors_len;
        ca.in_cert = TRUE;
        ca.dn_len = 0;
        ta = &ca.anchors[ca.anchors_len - 1];
        ta->dn.data = NULL;
        break;
      case BR_PEM_END_OBJ:
        if(!ca.in_cert)
          break;
        ca.in_cert = FALSE;
        if(br_x509_decoder_last_error(&ca.xc)) {
          ca.err = CURLE_SSL_CACERT_BADFILE;
          goto fail;
        }
        ta->flags = 0;
        if(br_x509_decoder_isCA(&ca.xc))
          ta->flags |= BR_X509_TA_CA;
        pkey = br_x509_decoder_get_pkey(&ca.xc);
        if(!pkey) {
          ca.err = CURLE_SSL_CACERT_BADFILE;
          goto fail;
        }
        ta->pkey = *pkey;

        /* calculate space needed for trust anchor data */
        ta_size = ca.dn_len;
        switch(pkey->key_type) {
        case BR_KEYTYPE_RSA:
          ta_size += pkey->key.rsa.nlen + pkey->key.rsa.elen;
          break;
        case BR_KEYTYPE_EC:
          ta_size += pkey->key.ec.qlen;
          break;
        default:
          ca.err = CURLE_FAILED_INIT;
          goto fail;
        }

        /* fill in trust anchor DN and public key data */
        ta->dn.data = malloc(ta_size);
        if(!ta->dn.data) {
          ca.err = CURLE_OUT_OF_MEMORY;
          goto fail;
        }
        memcpy(ta->dn.data, ca.dn, ca.dn_len);
        ta->dn.len = ca.dn_len;
        switch(pkey->key_type) {
        case BR_KEYTYPE_RSA:
          ta->pkey.key.rsa.n = ta->dn.data + ta->dn.len;
          memcpy(ta->pkey.key.rsa.n, pkey->key.rsa.n, pkey->key.rsa.nlen);
          ta->pkey.key.rsa.e = ta->pkey.key.rsa.n + ta->pkey.key.rsa.nlen;
          memcpy(ta->pkey.key.rsa.e, pkey->key.rsa.e, pkey->key.rsa.elen);
          break;
        case BR_KEYTYPE_EC:
          ta->pkey.key.ec.q = ta->dn.data + ta->dn.len;
          memcpy(ta->pkey.key.ec.q, pkey->key.ec.q, pkey->key.ec.qlen);
          break;
        }
        break;
      default:
        ca.err = CURLE_SSL_CACERT_BADFILE;
        goto fail;
      }
    }
  }
  if(ferror(fp))
    ca.err = CURLE_READ_ERROR;

fail:
  fclose(fp);
  if(ca.err == CURLE_OK) {
    *anchors = ca.anchors;
    *anchors_len = ca.anchors_len;
  }
  else {
    for(i = 0; i < ca.anchors_len; ++i)
      free(ca.anchors[i].dn.data);
    free(ca.anchors);
  }

  return ca.err;
}


static int gtls_shutdown(struct Curl_easy *data, struct connectdata *conn,
                         int sockindex)
{
  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
  struct ssl_backend_data *backend = connssl->backend;
  int retval = 0;

#ifndef CURL_DISABLE_FTP
  /* This has only been tested on the proftpd server, and the mod_tls code
     sends a close notify alert without waiting for a close notify alert in
     response. Thus we wait for a close notify alert from the server, but
     we do not send one. Let's hope other servers do the same... */

  if(data->set.ftp_ccc == CURLFTPSSL_CCC_ACTIVE)
    gnutls_bye(backend->session, GNUTLS_SHUT_WR);
#endif

  if(backend->session) {
    ssize_t result;
    bool done = FALSE;
    char buf[120];

    while(!done) {
      int what = SOCKET_READABLE(conn->sock[sockindex],
                                 SSL_SHUTDOWN_TIMEOUT);
      if(what > 0) {
        /* Something to read, let's do it and hope that it is the close
           notify alert from the server */
        result = gnutls_record_recv(backend->session,
                                    buf, sizeof(buf));
        switch(result) {
        case 0:
          /* This is the expected response. There was no data but only
             the close notify alert */
          done = TRUE;
          break;
        case GNUTLS_E_AGAIN:
        case GNUTLS_E_INTERRUPTED:
          infof(data, "GNUTLS_E_AGAIN || GNUTLS_E_INTERRUPTED\n");
          break;
        default:
          retval = -1;
          done = TRUE;
          break;
        }
      }
      else if(0 == what) {
        /* timeout */
        failf(data, "SSL shutdown timeout");
        done = TRUE;
      }
      else {
        /* anything that gets here is fatally bad */
        failf(data, "select/poll on SSL socket, errno: %d", SOCKERRNO);
        retval = -1;
        done = TRUE;
      }
    }
    gnutls_deinit(backend->session);
  }
  gnutls_certificate_free_credentials(backend->cred);

#ifdef HAVE_GNUTLS_SRP
  if(SSL_SET_OPTION(authtype) == CURL_TLSAUTH_SRP
     && SSL_SET_OPTION(username) != NULL)
    gnutls_srp_free_client_credentials(backend->srp_client_cred);
#endif

  backend->cred = NULL;
  backend->session = NULL;

  return retval;
}


static void gtls_cleanup(void)
{
  if(gtls_inited) {
    gnutls_global_deinit();
    gtls_inited = FALSE;
  }
}


static void x509_start_chain(const br_x509_class **ctx,
                             const char *server_name)
{
  struct x509_context *x509 = (struct x509_context *)ctx;

  if(!x509->verifyhost)
    server_name = NULL;
  x509->minimal.vtable->start_chain(&x509->minimal.vtable, server_name);
}


static bool gtls_data_pending(const struct connectdata *conn,
                              int connindex)
{
  const struct ssl_connect_data *connssl = &conn->ssl[connindex];
  bool res = FALSE;
  struct ssl_backend_data *backend = connssl->backend;
  if(backend->session &&
     0 != gnutls_record_check_pending(backend->session))
    res = TRUE;

#ifndef CURL_DISABLE_PROXY
  connssl = &conn->proxy_ssl[connindex];
  backend = connssl->backend;
  if(backend->session &&
     0 != gnutls_record_check_pending(backend->session))
    res = TRUE;
#endif

  return res;
}


static CURLcode bearssl_run_until(struct Curl_easy *data,
                                  struct connectdata *conn, int sockindex,
                                  unsigned target)
{
  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
  struct ssl_backend_data *backend = connssl->backend;
  curl_socket_t sockfd = conn->sock[sockindex];
  unsigned state;
  unsigned char *buf;
  size_t len;
  ssize_t ret;
  int err;

  for(;;) {
    state = br_ssl_engine_current_state(&backend->ctx.eng);
    if(state & BR_SSL_CLOSED) {
      err = br_ssl_engine_last_error(&backend->ctx.eng);
      switch(err) {
      case BR_ERR_OK:
        /* TLS close notify */
        if(connssl->state != ssl_connection_complete) {
          failf(data, "SSL: connection closed during handshake");
          return CURLE_SSL_CONNECT_ERROR;
        }
        return CURLE_OK;
      case BR_ERR_X509_EXPIRED:
        failf(data, "SSL: X.509 verification: "
              "certificate is expired or not yet valid");
        return CURLE_PEER_FAILED_VERIFICATION;
      case BR_ERR_X509_BAD_SERVER_NAME:
        failf(data, "SSL: X.509 verification: "
              "expected server name was not found in the chain");
        return CURLE_PEER_FAILED_VERIFICATION;
      case BR_ERR_X509_NOT_TRUSTED:
        failf(data, "SSL: X.509 verification: "
              "chain could not be linked to a trust anchor");
        return CURLE_PEER_FAILED_VERIFICATION;
      }
      /* X.509 errors are documented to have the range 32..63 */
      if(err >= 32 && err < 64)
        return CURLE_PEER_FAILED_VERIFICATION;
      return CURLE_SSL_CONNECT_ERROR;
    }
    if(state & target)
      return CURLE_OK;
    if(state & BR_SSL_SENDREC) {
      buf = br_ssl_engine_sendrec_buf(&backend->ctx.eng, &len);
      ret = swrite(sockfd, buf, len);
      if(ret == -1) {
        if(SOCKERRNO == EAGAIN || SOCKERRNO == EWOULDBLOCK) {
          if(connssl->state != ssl_connection_complete)
            connssl->connecting_state = ssl_connect_2_writing;
          return CURLE_AGAIN;
        }
        return CURLE_WRITE_ERROR;
      }
      br_ssl_engine_sendrec_ack(&backend->ctx.eng, ret);
    }
    else if(state & BR_SSL_RECVREC) {
      buf = br_ssl_engine_recvrec_buf(&backend->ctx.eng, &len);
      ret = sread(sockfd, buf, len);
      if(ret == 0) {
        failf(data, "SSL: EOF without close notify");
        return CURLE_READ_ERROR;
      }
      if(ret == -1) {
        if(SOCKERRNO == EAGAIN || SOCKERRNO == EWOULDBLOCK) {
          if(connssl->state != ssl_connection_complete)
            connssl->connecting_state = ssl_connect_2_reading;
          return CURLE_AGAIN;
        }
        return CURLE_READ_ERROR;
      }
      br_ssl_engine_recvrec_ack(&backend->ctx.eng, ret);
    }
  }
}


static void mbed_debug(void *context, int level, const char *f_name,
                       int line_nb, const char *line)
{
  struct Curl_easy *data = NULL;

  if(!context)
    return;

  data = (struct Curl_easy *)context;

  infof(data, "%s", line);
  (void) level;
}


static void x509_push(void *ctx, const void *buf, size_t len)
{
  struct cafile_parser *ca = ctx;

  if(ca->in_cert)
    br_x509_decoder_push(&ca->xc, buf, len);
}


static ssize_t wolfssl_recv(struct Curl_easy *data,
                            int num,
                            char *buf,
                            size_t buffersize,
                            CURLcode *curlcode)
{
  struct connectdata *conn = data->conn;
  struct ssl_connect_data *connssl = &conn->ssl[num];
  struct ssl_backend_data *backend = connssl->backend;
  char error_buffer[WOLFSSL_MAX_ERROR_SZ];
  int buffsize = (buffersize > (size_t)INT_MAX) ? INT_MAX : (int)buffersize;
  int nread = SSL_read(backend->handle, buf, buffsize);

  if(nread < 0) {
    int err = SSL_get_error(backend->handle, nread);

    switch(err) {
    case SSL_ERROR_ZERO_RETURN: /* no more data */
      break;
    case SSL_ERROR_WANT_READ:
    case SSL_ERROR_WANT_WRITE:
      /* there's data pending, re-invoke SSL_read() */
      *curlcode = CURLE_AGAIN;
      return -1;
    default:
      failf(data, "SSL read: %s, errno %d",
            ERR_error_string(err, error_buffer), SOCKERRNO);
      *curlcode = CURLE_RECV_ERROR;
      return -1;
    }
  }
  return nread;
}


static void x509_start_cert(const br_x509_class **ctx, uint32_t length)
{
  struct x509_context *x509 = (struct x509_context *)ctx;

  x509->minimal.vtable->start_cert(&x509->minimal.vtable, length);
}


mbed_connect_step2(struct Curl_easy *data, struct connectdata *conn,
                   int sockindex)
{
  int ret;
  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
  struct ssl_backend_data *backend = connssl->backend;
  const mbedtls_x509_crt *peercert;
#ifndef CURL_DISABLE_PROXY
  const char * const pinnedpubkey = SSL_IS_PROXY() ?
    data->set.str[STRING_SSL_PINNEDPUBLICKEY_PROXY] :
    data->set.str[STRING_SSL_PINNEDPUBLICKEY];
#else
  const char * const pinnedpubkey =
    data->set.str[STRING_SSL_PINNEDPUBLICKEY];
#endif

  conn->recv[sockindex] = mbed_recv;
  conn->send[sockindex] = mbed_send;

  ret = mbedtls_ssl_handshake(&backend->ssl);

  if(ret == MBEDTLS_ERR_SSL_WANT_READ) {
    connssl->connecting_state = ssl_connect_2_reading;
    return CURLE_OK;
  }
  else if(ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
    connssl->connecting_state = ssl_connect_2_writing;
    return CURLE_OK;
  }
  else if(ret) {
    char errorbuf[128];
    errorbuf[0] = 0;
#ifdef MBEDTLS_ERROR_C
    mbedtls_strerror(ret, errorbuf, sizeof(errorbuf));
#endif /* MBEDTLS_ERROR_C */
    failf(data, "ssl_handshake returned - mbedTLS: (-0x%04X) %s",
          -ret, errorbuf);
    return CURLE_SSL_CONNECT_ERROR;
  }

  infof(data, "mbedTLS: Handshake complete, cipher is %s\n",
        mbedtls_ssl_get_ciphersuite(&backend->ssl)
    );

  ret = mbedtls_ssl_get_verify_result(&backend->ssl);

  if(!SSL_CONN_CONFIG(verifyhost))
    /* Ignore hostname errors if verifyhost is disabled */
    ret &= ~MBEDTLS_X509_BADCERT_CN_MISMATCH;

  if(ret && SSL_CONN_CONFIG(verifypeer)) {
    if(ret & MBEDTLS_X509_BADCERT_EXPIRED)
      failf(data, "Cert verify failed: BADCERT_EXPIRED");

    else if(ret & MBEDTLS_X509_BADCERT_REVOKED)
      failf(data, "Cert verify failed: BADCERT_REVOKED");

    else if(ret & MBEDTLS_X509_BADCERT_CN_MISMATCH)
      failf(data, "Cert verify failed: BADCERT_CN_MISMATCH");

    else if(ret & MBEDTLS_X509_BADCERT_NOT_TRUSTED)
      failf(data, "Cert verify failed: BADCERT_NOT_TRUSTED");

    else if(ret & MBEDTLS_X509_BADCERT_FUTURE)
      failf(data, "Cert verify failed: BADCERT_FUTURE");

    return CURLE_PEER_FAILED_VERIFICATION;
  }

  peercert = mbedtls_ssl_get_peer_cert(&backend->ssl);

  if(peercert && data->set.verbose) {
    const size_t bufsize = 16384;
    char *buffer = malloc(bufsize);

    if(!buffer)
      return CURLE_OUT_OF_MEMORY;

    if(mbedtls_x509_crt_info(buffer, bufsize, "* ", peercert) > 0)
      infof(data, "Dumping cert info:\n%s\n", buffer);
    else
      infof(data, "Unable to dump certificate information.\n");

    free(buffer);
  }

  if(pinnedpubkey) {
    int size;
    CURLcode result;
    mbedtls_x509_crt *p;
    unsigned char pubkey[PUB_DER_MAX_BYTES];

    if(!peercert || !peercert->raw.p || !peercert->raw.len) {
      failf(data, "Failed due to missing peer certificate");
      return CURLE_SSL_PINNEDPUBKEYNOTMATCH;
    }

    p = calloc(1, sizeof(*p));

    if(!p)
      return CURLE_OUT_OF_MEMORY;

    mbedtls_x509_crt_init(p);

    /* Make a copy of our const peercert because mbedtls_pk_write_pubkey_der
       needs a non-const key, for now.
       https://github.com/ARMmbed/mbedtls/issues/396 */
    if(mbedtls_x509_crt_parse_der(p, peercert->raw.p, peercert->raw.len)) {
      failf(data, "Failed copying peer certificate");
      mbedtls_x509_crt_free(p);
      free(p);
      return CURLE_SSL_PINNEDPUBKEYNOTMATCH;
    }

    size = mbedtls_pk_write_pubkey_der(&p->pk, pubkey, PUB_DER_MAX_BYTES);

    if(size <= 0) {
      failf(data, "Failed copying public key from peer certificate");
      mbedtls_x509_crt_free(p);
      free(p);
      return CURLE_SSL_PINNEDPUBKEYNOTMATCH;
    }

    /* mbedtls_pk_write_pubkey_der writes data at the end of the buffer. */
    result = Curl_pin_peer_pubkey(data,
                                  pinnedpubkey,
                                  &pubkey[PUB_DER_MAX_BYTES - size], size);
    if(result) {
      mbedtls_x509_crt_free(p);
      free(p);
      return result;
    }

    mbedtls_x509_crt_free(p);
    free(p);
  }

#ifdef HAS_ALPN
  if(conn->bits.tls_enable_alpn) {
    const char *next_protocol = mbedtls_ssl_get_alpn_protocol(&backend->ssl);

    if(next_protocol) {
      infof(data, "ALPN, server accepted to use %s\n", next_protocol);
#ifdef USE_NGHTTP2
      if(!strncmp(next_protocol, NGHTTP2_PROTO_VERSION_ID,
                  NGHTTP2_PROTO_VERSION_ID_LEN) &&
         !next_protocol[NGHTTP2_PROTO_VERSION_ID_LEN]) {
        conn->negnpn = CURL_HTTP_VERSION_2;
      }
      else
#endif
        if(!strncmp(next_protocol, ALPN_HTTP_1_1, ALPN_HTTP_1_1_LENGTH) &&
           !next_protocol[ALPN_HTTP_1_1_LENGTH]) {
          conn->negnpn = CURL_HTTP_VERSION_1_1;
        }
    }
    else {
      infof(data, "ALPN, server did not agree to a protocol\n");
    }
    Curl_multiuse_state(data, conn->negnpn == CURL_HTTP_VERSION_2 ?
                        BUNDLE_MULTIPLEX : BUNDLE_NO_MULTIUSE);
  }
#endif

  connssl->connecting_state = ssl_connect_3;
  infof(data, "SSL connected\n");

  return CURLE_OK;
}


static void *bearssl_get_internals(struct ssl_connect_data *connssl,
                                   CURLINFO info UNUSED_PARAM)
{
  struct ssl_backend_data *backend = connssl->backend;
  return &backend->ctx;
}


mesalink_send(struct Curl_easy *data, int sockindex, const void *mem,
              size_t len, CURLcode *curlcode)
{
  struct connectdata *conn = data->conn;
  struct ssl_connect_data *connssl = &conn->ssl[sockindex];
  char error_buffer[MESALINK_MAX_ERROR_SZ];
  int memlen = (len > (size_t)INT_MAX) ? INT_MAX : (int)len;
  int rc = SSL_write(BACKEND->handle, mem, memlen);

  if(rc < 0) {
    int err = SSL_get_error(BACKEND->handle, rc);
    switch(err) {
    case SSL_ERROR_WANT_READ:
    case SSL_ERROR_WANT_WRITE:
      /* there's data pending, re-invoke SSL_write() */
      *curlcode = CURLE_AGAIN;
      return -1;
    default:
      failf(data,
            "SSL write: %s, errno %d",
            ERR_error_string_n(err, error_buffer, sizeof(error_buffer)),
            SOCKERRNO);
      *curlcode = CURLE_SEND_ERROR;
      return -1;
    }
  }
  return rc;
}


ecc_mod_mul (const struct ecc_modulo *m, mp_limb_t *rp,
	     const mp_limb_t *ap, const mp_limb_t *bp, mp_limb_t *tp)
{
  mpn_mul_n (tp, ap, bp, m->size);
  m->reduce (m, rp, tp);
}


ecc_mod_sub (const struct ecc_modulo *m, mp_limb_t *rp,
	     const mp_limb_t *ap, const mp_limb_t *bp)
{
  mp_limb_t cy;
  cy = mpn_sub_n (rp, ap, bp, m->size);
  cy = mpn_cnd_sub_n (cy, rp, rp, m->B, m->size);
  cy = mpn_cnd_sub_n (cy, rp, rp, m->B, m->size);
  assert (cy == 0);
}


ecc_ecdsa_verify (const struct ecc_curve *ecc,
		  const mp_limb_t *pp, /* Public key */
		  size_t length, const uint8_t *digest,
		  const mp_limb_t *rp, const mp_limb_t *sp,
		  mp_limb_t *scratch)
{
  /* Procedure, according to RFC 6090, "KT-I". q denotes the group
     order.

     1. Check 0 < r, s < q.

     2. s' <-- s^{-1}  (mod q)

     3. u1  <-- h * s' (mod q)

     4. u2  <-- r * s' (mod q)

     5. R = u1 G + u2 Y

     6. Signature is valid if R_x = r (mod q).
  */

#define P2 scratch
#define u1 (scratch + 3*ecc->p.size)
#define u2 (scratch + 4*ecc->p.size)

#define P1 (scratch + 4*ecc->p.size)
#define sinv (scratch)
#define hp (scratch + ecc->p.size)

  if (! (ecdsa_in_range (ecc, rp)
	 && ecdsa_in_range (ecc, sp)))
    return 0;

  /* FIXME: Micro optimizations: Either simultaneous multiplication.
     Or convert to projective coordinates (can be done without
     division, I think), and write an ecc_add_ppp. */

  /* Compute sinv */
  ecc->q.invert (&ecc->q, sinv, sp, sinv + ecc->p.size);

  /* u1 = h / s, P1 = u1 * G */
  ecc_hash (&ecc->q, hp, length, digest);
  ecc_mod_mul_canonical (&ecc->q, u1, hp, sinv, u1);

  /* u2 = r / s, P2 = u2 * Y */
  ecc_mod_mul_canonical (&ecc->q, u2, rp, sinv, u2);

   /* Total storage: 5*ecc->p.size + ecc->mul_itch */
  ecc->mul (ecc, P2, u2, pp, u2 + ecc->p.size);

  /* u = 0 can happen only if h = 0 or h = q, which is extremely
     unlikely. */
  if (!mpn_zero_p (u1, ecc->p.size))
    {
      /* Total storage: 7*ecc->p.size + ecc->mul_g_itch (ecc->p.size) */
      ecc->mul_g (ecc, P1, u1, P1 + 3*ecc->p.size);

      /* NOTE: ecc_add_jjj and/or ecc_j_to_a will produce garbage in
	 case u1 G = +/- u2 V. However, anyone who gets his or her
	 hands on a signature where this happens during verification,
	 can also get the private key as z = +/- u1 / u_2 (mod q). And
	 then it doesn't matter very much if verification of
	 signatures with that key succeeds or fails.

	 u1 G = - u2 V can never happen for a correctly generated
	 signature, since it implies k = 0.

	 u1 G = u2 V is possible, if we are unlucky enough to get h /
	 s_1 = z. Hitting that is about as unlikely as finding the
	 private key by guessing.
       */
      /* Total storage: 6*ecc->p.size + ecc->add_hhh_itch */
      ecc->add_hhh (ecc, P2, P2, P1, P1 + 3*ecc->p.size);
    }
  /* x coordinate only, modulo q */
  ecc->h_to_a (ecc, 2, P1, P2, P1 + 3*ecc->p.size);

  return (mpn_cmp (rp, P1, ecc->p.size) == 0);
#undef P2
#undef P1
#undef sinv
#undef u2
#undef hp
#undef u1
}


test_ecdsa (const struct ecc_curve *ecc,
	    /* Private key */
	    const char *sz,
	    /* Random nonce */
	    const char *sk,
	    /* Hash */
	    const struct tstring *h,
	    /* Expected signature */
	    const char *r, const char *s)
{
  struct dsa_signature ref;
  mpz_t z;
  mpz_t k;
  mp_limb_t *rp = xalloc_limbs (ecc->p.size);
  mp_limb_t *sp = xalloc_limbs (ecc->p.size);
  mp_limb_t *scratch = xalloc_limbs (ecc_ecdsa_sign_itch (ecc));

  dsa_signature_init (&ref);

  mpz_init_set_str (z, sz, 16);
  mpz_init_set_str (k, sk, 16);

  ecc_ecdsa_sign (ecc, mpz_limbs_read_n (z, ecc->p.size),
		  mpz_limbs_read_n (k, ecc->p.size),
		  h->length, h->data, rp, sp, scratch);

  mpz_set_str (ref.r, r, 16);
  mpz_set_str (ref.s, s, 16);

  if (mpz_limbs_cmp (ref.r, rp, ecc->p.size) != 0
      || mpz_limbs_cmp (ref.s, sp, ecc->p.size) != 0)
    {
      fprintf (stderr, "_ecdsa_sign failed, bit_size = %u\n", ecc->p.bit_size);
      fprintf (stderr, "r     = ");
      write_mpn (stderr, 16, rp, ecc->p.size);
      fprintf (stderr, "\ns     = ");
      write_mpn (stderr, 16, sp, ecc->p.size);
      fprintf (stderr, "\nref.r = ");
      mpz_out_str (stderr, 16, ref.r);
      fprintf (stderr, "\nref.s = ");
      mpz_out_str (stderr, 16, ref.s);
      fprintf (stderr, "\n");
      abort();
    }

  free (rp);
  free (sp);
  free (scratch);

  dsa_signature_clear (&ref);
  mpz_clear (k);
  mpz_clear (z);
}


static int fixup_bpf_calls(struct bpf_verifier_env *env)
{
	struct bpf_prog *prog = env->prog;
	bool expect_blinding = bpf_jit_blinding_enabled(prog);
	struct bpf_insn *insn = prog->insnsi;
	const struct bpf_func_proto *fn;
	const int insn_cnt = prog->len;
	const struct bpf_map_ops *ops;
	struct bpf_insn_aux_data *aux;
	struct bpf_insn insn_buf[16];
	struct bpf_prog *new_prog;
	struct bpf_map *map_ptr;
	int i, ret, cnt, delta = 0;

	for (i = 0; i < insn_cnt; i++, insn++) {
		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
			struct bpf_insn *patchlet;
			struct bpf_insn chk_and_div[] = {
				/* [R,W]x div 0 -> 0 */
				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
					     BPF_JNE | BPF_K, insn->src_reg,
					     0, 2, 0),
				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
				*insn,
			};
			struct bpf_insn chk_and_mod[] = {
				/* [R,W]x mod 0 -> [R,W]x */
				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
					     BPF_JEQ | BPF_K, insn->src_reg,
					     0, 1 + (is64 ? 0 : 1), 0),
				*insn,
				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
				BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
			};

			patchlet = isdiv ? chk_and_div : chk_and_mod;
			cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
				      ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);

			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

		if (BPF_CLASS(insn->code) == BPF_LD &&
		    (BPF_MODE(insn->code) == BPF_ABS ||
		     BPF_MODE(insn->code) == BPF_IND)) {
			cnt = env->ops->gen_ld_abs(insn, insn_buf);
			if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
				verbose(env, "bpf verifier is misconfigured\n");
				return -EINVAL;
			}

			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
			struct bpf_insn insn_buf[16];
			struct bpf_insn *patch = &insn_buf[0];
			bool issrc, isneg;
			u32 off_reg;

			aux = &env->insn_aux_data[i + delta];
			if (!aux->alu_state ||
			    aux->alu_state == BPF_ALU_NON_POINTER)
				continue;

			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
				BPF_ALU_SANITIZE_SRC;

			off_reg = issrc ? insn->src_reg : insn->dst_reg;
			if (isneg)
				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
			*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1);
			*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
			*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
			*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
			*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
			if (issrc) {
				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX,
							 off_reg);
				insn->src_reg = BPF_REG_AX;
			} else {
				*patch++ = BPF_ALU64_REG(BPF_AND, off_reg,
							 BPF_REG_AX);
			}
			if (isneg)
				insn->code = insn->code == code_add ?
					     code_sub : code_add;
			*patch++ = *insn;
			if (issrc && isneg)
				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
			cnt = patch - insn_buf;

			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

		if (insn->code != (BPF_JMP | BPF_CALL))
			continue;
		if (insn->src_reg == BPF_PSEUDO_CALL)
			continue;

		if (insn->imm == BPF_FUNC_get_route_realm)
			prog->dst_needed = 1;
		if (insn->imm == BPF_FUNC_get_prandom_u32)
			bpf_user_rnd_init_once();
		if (insn->imm == BPF_FUNC_override_return)
			prog->kprobe_override = 1;
		if (insn->imm == BPF_FUNC_tail_call) {
			/* If we tail call into other programs, we
			 * cannot make any assumptions since they can
			 * be replaced dynamically during runtime in
			 * the program array.
			 */
			prog->cb_access = 1;
			if (!allow_tail_call_in_subprogs(env))
				prog->aux->stack_depth = MAX_BPF_STACK;
			prog->aux->max_pkt_offset = MAX_PACKET_OFF;

			/* mark bpf_tail_call as different opcode to avoid
			 * conditional branch in the interpeter for every normal
			 * call and to prevent accidental JITing by JIT compiler
			 * that doesn't support bpf_tail_call yet
			 */
			insn->imm = 0;
			insn->code = BPF_JMP | BPF_TAIL_CALL;

			aux = &env->insn_aux_data[i + delta];
			if (env->bpf_capable && !expect_blinding &&
			    prog->jit_requested &&
			    !bpf_map_key_poisoned(aux) &&
			    !bpf_map_ptr_poisoned(aux) &&
			    !bpf_map_ptr_unpriv(aux)) {
				struct bpf_jit_poke_descriptor desc = {
					.reason = BPF_POKE_REASON_TAIL_CALL,
					.tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
					.tail_call.key = bpf_map_key_immediate(aux),
					.insn_idx = i + delta,
				};

				ret = bpf_jit_add_poke_descriptor(prog, &desc);
				if (ret < 0) {
					verbose(env, "adding tail call poke descriptor failed\n");
					return ret;
				}

				insn->imm = ret + 1;
				continue;
			}

			if (!bpf_map_ptr_unpriv(aux))
				continue;

			/* instead of changing every JIT dealing with tail_call
			 * emit two extra insns:
			 * if (index >= max_entries) goto out;
			 * index &= array->index_mask;
			 * to avoid out-of-bounds cpu speculation
			 */
			if (bpf_map_ptr_poisoned(aux)) {
				verbose(env, "tail_call abusing map_ptr\n");
				return -EINVAL;
			}

			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
						  map_ptr->max_entries, 2);
			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
						    container_of(map_ptr,
								 struct bpf_array,
								 map)->index_mask);
			insn_buf[2] = *insn;
			cnt = 3;
			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
		 * and other inlining handlers are currently limited to 64 bit
		 * only.
		 */
		if (prog->jit_requested && BITS_PER_LONG == 64 &&
		    (insn->imm == BPF_FUNC_map_lookup_elem ||
		     insn->imm == BPF_FUNC_map_update_elem ||
		     insn->imm == BPF_FUNC_map_delete_elem ||
		     insn->imm == BPF_FUNC_map_push_elem   ||
		     insn->imm == BPF_FUNC_map_pop_elem    ||
		     insn->imm == BPF_FUNC_map_peek_elem)) {
			aux = &env->insn_aux_data[i + delta];
			if (bpf_map_ptr_poisoned(aux))
				goto patch_call_imm;

			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
			ops = map_ptr->ops;
			if (insn->imm == BPF_FUNC_map_lookup_elem &&
			    ops->map_gen_lookup) {
				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
				if (cnt == -EOPNOTSUPP)
					goto patch_map_ops_generic;
				if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
					verbose(env, "bpf verifier is misconfigured\n");
					return -EINVAL;
				}

				new_prog = bpf_patch_insn_data(env, i + delta,
							       insn_buf, cnt);
				if (!new_prog)
					return -ENOMEM;

				delta    += cnt - 1;
				env->prog = prog = new_prog;
				insn      = new_prog->insnsi + i + delta;
				continue;
			}

			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
				     (void *(*)(struct bpf_map *map, void *key))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
				     (int (*)(struct bpf_map *map, void *key))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
				     (int (*)(struct bpf_map *map, void *key, void *value,
					      u64 flags))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
				     (int (*)(struct bpf_map *map, void *value,
					      u64 flags))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
				     (int (*)(struct bpf_map *map, void *value))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
				     (int (*)(struct bpf_map *map, void *value))NULL));
patch_map_ops_generic:
			switch (insn->imm) {
			case BPF_FUNC_map_lookup_elem:
				insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_update_elem:
				insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_delete_elem:
				insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_push_elem:
				insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_pop_elem:
				insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_peek_elem:
				insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
					    __bpf_call_base;
				continue;
			}

			goto patch_call_imm;
		}

		if (prog->jit_requested && BITS_PER_LONG == 64 &&
		    insn->imm == BPF_FUNC_jiffies64) {
			struct bpf_insn ld_jiffies_addr[2] = {
				BPF_LD_IMM64(BPF_REG_0,
					     (unsigned long)&jiffies),
			};

			insn_buf[0] = ld_jiffies_addr[0];
			insn_buf[1] = ld_jiffies_addr[1];
			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
						  BPF_REG_0, 0);
			cnt = 3;

			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
						       cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

patch_call_imm:
		fn = env->ops->get_func_proto(insn->imm, env->prog);
		/* all functions that have prototype and verifier allowed
		 * programs to call them, must be real in-kernel functions
		 */
		if (!fn->func) {
			verbose(env,
				"kernel subsystem misconfigured func %s#%d\n",
				func_id_name(insn->imm), insn->imm);
			return -EFAULT;
		}
		insn->imm = fn->func - __bpf_call_base;
	}

	/* Since poke tab is now finalized, publish aux to tracker. */
	for (i = 0; i < prog->aux->size_poke_tab; i++) {
		map_ptr = prog->aux->poke_tab[i].tail_call.map;
		if (!map_ptr->ops->map_poke_track ||
		    !map_ptr->ops->map_poke_untrack ||
		    !map_ptr->ops->map_poke_run) {
			verbose(env, "bpf verifier is misconfigured\n");
			return -EINVAL;
		}

		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
		if (ret < 0) {
			verbose(env, "tracking tail call prog failed\n");
			return ret;
		}
	}

	return 0;
}


static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
				int access_size, bool zero_size_allowed,
				struct bpf_call_arg_meta *meta)
{
	struct bpf_reg_state *reg = reg_state(env, regno);
	struct bpf_func_state *state = func(env, reg);
	int err, min_off, max_off, i, j, slot, spi;

	if (tnum_is_const(reg->var_off)) {
		min_off = max_off = reg->var_off.value + reg->off;
		err = __check_stack_boundary(env, regno, min_off, access_size,
					     zero_size_allowed);
		if (err)
			return err;
	} else {
		/* Variable offset is prohibited for unprivileged mode for
		 * simplicity since it requires corresponding support in
		 * Spectre masking for stack ALU.
		 * See also retrieve_ptr_limit().
		 */
		if (!env->bypass_spec_v1) {
			char tn_buf[48];

			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
			verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
				regno, tn_buf);
			return -EACCES;
		}
		/* Only initialized buffer on stack is allowed to be accessed
		 * with variable offset. With uninitialized buffer it's hard to
		 * guarantee that whole memory is marked as initialized on
		 * helper return since specific bounds are unknown what may
		 * cause uninitialized stack leaking.
		 */
		if (meta && meta->raw_mode)
			meta = NULL;

		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
		    reg->smax_value <= -BPF_MAX_VAR_OFF) {
			verbose(env, "R%d unbounded indirect variable offset stack access\n",
				regno);
			return -EACCES;
		}
		min_off = reg->smin_value + reg->off;
		max_off = reg->smax_value + reg->off;
		err = __check_stack_boundary(env, regno, min_off, access_size,
					     zero_size_allowed);
		if (err) {
			verbose(env, "R%d min value is outside of stack bound\n",
				regno);
			return err;
		}
		err = __check_stack_boundary(env, regno, max_off, access_size,
					     zero_size_allowed);
		if (err) {
			verbose(env, "R%d max value is outside of stack bound\n",
				regno);
			return err;
		}
	}

	if (meta && meta->raw_mode) {
		meta->access_size = access_size;
		meta->regno = regno;
		return 0;
	}

	for (i = min_off; i < max_off + access_size; i++) {
		u8 *stype;

		slot = -i - 1;
		spi = slot / BPF_REG_SIZE;
		if (state->allocated_stack <= slot)
			goto err;
		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
		if (*stype == STACK_MISC)
			goto mark;
		if (*stype == STACK_ZERO) {
			/* helper can write anything into the stack */
			*stype = STACK_MISC;
			goto mark;
		}

		if (state->stack[spi].slot_type[0] == STACK_SPILL &&
		    state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID)
			goto mark;

		if (state->stack[spi].slot_type[0] == STACK_SPILL &&
		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
		     env->allow_ptr_leaks)) {
			__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
			for (j = 0; j < BPF_REG_SIZE; j++)
				state->stack[spi].slot_type[j] = STACK_MISC;
			goto mark;
		}

err:
		if (tnum_is_const(reg->var_off)) {
			verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
				min_off, i - min_off, access_size);
		} else {
			char tn_buf[48];

			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
			verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
				tn_buf, i - min_off, access_size);
		}
		return -EACCES;
mark:
		/* reading any byte out of 8-byte 'spill_slot' will cause
		 * the whole slot to be marked as 'read'
		 */
		mark_reg_read(env, &state->stack[spi].spilled_ptr,
			      state->stack[spi].spilled_ptr.parent,
			      REG_LIVE_READ64);
	}
	return update_stack_depth(env, state, min_off);
}


static void mark_ptr_or_null_reg(struct bpf_func_state *state,
				 struct bpf_reg_state *reg, u32 id,
				 bool is_null)
{
	if (reg_type_may_be_null(reg->type) && reg->id == id &&
	    !WARN_ON_ONCE(!reg->id)) {
		/* Old offset (both fixed and variable parts) should
		 * have been known-zero, because we don't allow pointer
		 * arithmetic on pointers that might be NULL.
		 */
		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
				 !tnum_equals_const(reg->var_off, 0) ||
				 reg->off)) {
			__mark_reg_known_zero(reg);
			reg->off = 0;
		}
		if (is_null) {
			reg->type = SCALAR_VALUE;
		} else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
			const struct bpf_map *map = reg->map_ptr;

			if (map->inner_map_meta) {
				reg->type = CONST_PTR_TO_MAP;
				reg->map_ptr = map->inner_map_meta;
			} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
				reg->type = PTR_TO_XDP_SOCK;
			} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
				   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
				reg->type = PTR_TO_SOCKET;
			} else {
				reg->type = PTR_TO_MAP_VALUE;
			}
		} else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
			reg->type = PTR_TO_SOCKET;
		} else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
			reg->type = PTR_TO_SOCK_COMMON;
		} else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
			reg->type = PTR_TO_TCP_SOCK;
		} else if (reg->type == PTR_TO_BTF_ID_OR_NULL) {
			reg->type = PTR_TO_BTF_ID;
		} else if (reg->type == PTR_TO_MEM_OR_NULL) {
			reg->type = PTR_TO_MEM;
		} else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) {
			reg->type = PTR_TO_RDONLY_BUF;
		} else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) {
			reg->type = PTR_TO_RDWR_BUF;
		}
		if (is_null) {
			/* We don't need id and ref_obj_id from this point
			 * onwards anymore, thus we should better reset it,
			 * so that state pruning has chances to take effect.
			 */
			reg->id = 0;
			reg->ref_obj_id = 0;
		} else if (!reg_may_point_to_spin_lock(reg)) {
			/* For not-NULL ptr, reg->ref_obj_id will be reset
			 * in release_reg_references().
			 *
			 * reg->id is still used by spin_lock ptr. Other
			 * than spin_lock ptr type, reg->id can be reset.
			 */
			reg->id = 0;
		}
	}
}


HRESULT Http::HrSetKeepAlive(int ulKeepAlive)
{
	m_ulKeepAlive = ulKeepAlive;
	return hrSuccess;
}


HRESULT Http::HrGetDepth(ULONG *ulDepth)
{
	std::string strDepth;
	/*
	 * Valid input: [0, 1, infinity]
	 */
	auto hr = HrGetHeaderValue("Depth", &strDepth);
	if (hr != hrSuccess)
		*ulDepth = 0; /* Default is no subfolders. Default should become a parameter. It is action dependent. */
	else if (strDepth == "infinity")
		*ulDepth = 2;
	else {
		*ulDepth = atoi(strDepth.c_str());
		if (*ulDepth > 1)
			*ulDepth = 1;
	}
	return hr;
}


HRESULT Http::HrRequestAuth(const std::string &strMsg)
{
	auto hr = HrResponseHeader(401, "Unauthorized");
	if (hr != hrSuccess)
		return hr;
	return HrResponseHeader("WWW-Authenticate", "Basic realm=\"" + strMsg + "\"");
}


HRESULT Http::HrGetHeaderValue(const std::string &strHeader, std::string *strValue)
{
	auto iHeader = mapHeaders.find(strHeader);
	if (iHeader == mapHeaders.cend())
		return MAPI_E_NOT_FOUND;
	*strValue = iHeader->second;
	return hrSuccess;
}


HRESULT HrParseURL(const std::string &strUrl, ULONG *lpulFlag, std::string *lpstrUrlUser, std::string *lpstrFolder)
{
	HRESULT hr = hrSuccess;
	std::string strService, strFolder, strUrlUser;
	ULONG ulFlag = 0;
	auto vcUrlTokens = tokenize(strUrl, L'/', true);
	decltype(vcUrlTokens)::const_iterator iterToken;
	if (vcUrlTokens.empty())
		// root should be present, no flags are set. mostly used on OPTIONS command
		goto exit;

	if (vcUrlTokens.back().rfind(".ics") != std::string::npos)
		// Guid is retrieved using StripGuid().
		vcUrlTokens.pop_back();
	else
		// request is for folder not a calendar entry
		ulFlag |= REQ_COLLECTION;
	if (vcUrlTokens.empty())
		goto exit;
	if (vcUrlTokens.size() > 3) {
		// sub folders are not allowed
		hr = MAPI_E_TOO_COMPLEX;
		goto exit;
	}

	iterToken = vcUrlTokens.cbegin();
	//change case of Service name ICAL -> ical CALDaV ->caldav
	strService = strToLower(*iterToken++);
	if (strService == "ical")
		ulFlag |= SERVICE_ICAL;
	else if (strService == "caldav")
		ulFlag |= SERVICE_CALDAV;
	else
		ulFlag |= SERVICE_UNKNOWN;
	if (iterToken == vcUrlTokens.cend())
		goto exit;

	//change case of folder owner USER -> user, UseR -> user
	strUrlUser = strToLower(*iterToken++);
	// check if the request is for public folders and set the bool flag
	// @note: request for public folder not have user's name in the url
	if (strUrlUser == "public")
		ulFlag |= REQ_PUBLIC;
	if (iterToken == vcUrlTokens.cend())
		goto exit;
	// @todo subfolder/folder/ is not allowed! only subfolder/item.ics
	for (; iterToken != vcUrlTokens.end(); ++iterToken)
		strFolder = strFolder + *iterToken + "/";
	strFolder.erase(strFolder.length() - 1);
exit:
	if (lpulFlag)
		*lpulFlag = ulFlag;
	if (lpstrUrlUser)
		*lpstrUrlUser = std::move(strUrlUser);
	if (lpstrFolder)
		*lpstrFolder = std::move(strFolder);
	return hr;
}


nm_utils_cmp_connection_by_autoconnect_priority(NMConnection *a, NMConnection *b)
{
    NMSettingConnection *a_s_con;
    NMSettingConnection *b_s_con;
    int                  a_ap, b_ap;
    gboolean             can_autoconnect;

    if (a == b)
        return 0;
    if (!a)
        return 1;
    if (!b)
        return -1;

    a_s_con = nm_connection_get_setting_connection(a);
    b_s_con = nm_connection_get_setting_connection(b);

    if (!a_s_con)
        return !b_s_con ? 0 : 1;
    if (!b_s_con)
        return -1;

    can_autoconnect = !!nm_setting_connection_get_autoconnect(a_s_con);
    if (can_autoconnect != (!!nm_setting_connection_get_autoconnect(b_s_con)))
        return can_autoconnect ? -1 : 1;

    if (can_autoconnect) {
        a_ap = nm_setting_connection_get_autoconnect_priority(a_s_con);
        b_ap = nm_setting_connection_get_autoconnect_priority(b_s_con);
        if (a_ap != b_ap)
            return (a_ap > b_ap) ? -1 : 1;
    }

    return 0;
}


_nm_singleton_instance_register_destruction(GObject *instance)
{
    g_return_if_fail(G_IS_OBJECT(instance));

    /* Don't allow registration after shutdown. We only destroy the singletons
     * once. */
    g_return_if_fail(!_singletons_shutdown);

    g_object_weak_ref(instance, _nm_singleton_instance_weak_cb, NULL);

    _singletons = g_slist_prepend(_singletons, instance);
}


_is_reserved_ipv6_iid(const guint8 *iid)
{
    /* https://tools.ietf.org/html/rfc5453 */
    /* https://www.iana.org/assignments/ipv6-interface-ids/ipv6-interface-ids.xml */

    /* 0000:0000:0000:0000 (Subnet-Router Anycast [RFC4291]) */
    if (memcmp(iid, &nm_ip_addr_zero.addr6.s6_addr[8], 8) == 0)
        return TRUE;

    /* 0200:5EFF:FE00:0000 - 0200:5EFF:FE00:5212 (Reserved IPv6 Interface Identifiers corresponding to the IANA Ethernet Block [RFC4291])
     * 0200:5EFF:FE00:5213                       (Proxy Mobile IPv6 [RFC6543])
     * 0200:5EFF:FE00:5214 - 0200:5EFF:FEFF:FFFF (Reserved IPv6 Interface Identifiers corresponding to the IANA Ethernet Block [RFC4291]) */
    if (memcmp(iid, (const guint8[]){0x02, 0x00, 0x5E, 0xFF, 0xFE}, 5) == 0)
        return TRUE;

    /* FDFF:FFFF:FFFF:FF80 - FDFF:FFFF:FFFF:FFFF (Reserved Subnet Anycast Addresses [RFC2526]) */
    if (memcmp(iid, (const guint8[]){0xFD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 7) == 0) {
        if (iid[7] & 0x80)
            return TRUE;
    }

    return FALSE;
}


nm_utils_machine_id_is_fake(void)
{
    return _machine_id_get(TRUE)->is_fake;
}


nm_utils_kill_child_sync(pid_t       pid,
                         int         sig,
                         NMLogDomain log_domain,
                         const char *log_name,
                         int *       child_status,
                         guint32     wait_before_kill_msec,
                         guint32     sleep_duration_msec)
{
    int      status = 0, errsv = 0;
    pid_t    ret;
    gboolean success     = FALSE;
    gboolean was_waiting = FALSE, send_kill = FALSE;
    char     buf_exit[KC_EXIT_TO_STRING_BUF_SIZE];
    char     buf_wait[KC_WAITED_TO_STRING];
    gint64   wait_start_us;

    g_return_val_if_fail(pid > 0, FALSE);
    g_return_val_if_fail(log_name != NULL, FALSE);

    /* check if the child process already terminated... */
    ret = waitpid(pid, &status, WNOHANG);
    if (ret > 0) {
        nm_log_dbg(log_domain,
                   LOG_NAME_FMT ": process %ld already terminated %s",
                   LOG_NAME_ARGS,
                   (long) ret,
                   _kc_exit_to_string(buf_exit, status));
        success = TRUE;
        goto out;
    } else if (ret != 0) {
        errsv = errno;
        /* ECHILD means, the process is not a child/does not exist or it has SIGCHILD blocked. */
        if (errsv != ECHILD) {
            nm_log_err(LOGD_CORE | log_domain,
                       LOG_NAME_FMT ": unexpected error while waitpid: %s (%d)",
                       LOG_NAME_ARGS,
                       nm_strerror_native(errsv),
                       errsv);
            goto out;
        }
    }

    /* send first signal @sig */
    if (kill(pid, sig) != 0) {
        errsv = errno;
        /* ESRCH means, process does not exist or is already a zombie. */
        if (errsv != ESRCH) {
            nm_log_err(LOGD_CORE | log_domain,
                       LOG_NAME_FMT ": failed to send %s: %s (%d)",
                       LOG_NAME_ARGS,
                       _kc_signal_to_string(sig),
                       nm_strerror_native(errsv),
                       errsv);
        } else {
            /* let's try again with waitpid, probably there was a race... */
            ret = waitpid(pid, &status, 0);
            if (ret > 0) {
                nm_log_dbg(log_domain,
                           LOG_NAME_FMT ": process %ld already terminated %s",
                           LOG_NAME_ARGS,
                           (long) ret,
                           _kc_exit_to_string(buf_exit, status));
                success = TRUE;
            } else {
                errsv = errno;
                nm_log_err(LOGD_CORE | log_domain,
                           LOG_NAME_FMT ": failed due to unexpected return value %ld by waitpid "
                                        "(%s, %d) after sending %s",
                           LOG_NAME_ARGS,
                           (long) ret,
                           nm_strerror_native(errsv),
                           errsv,
                           _kc_signal_to_string(sig));
            }
        }
        goto out;
    }

    wait_start_us = nm_utils_get_monotonic_timestamp_usec();

    /* wait for the process to terminated... */
    if (sig != SIGKILL) {
        gint64 wait_until, now;
        gulong sleep_time, sleep_duration_usec;
        int    loop_count = 0;

        sleep_duration_usec = _sleep_duration_convert_ms_to_us(sleep_duration_msec);
        wait_until          = wait_before_kill_msec <= 0
                                  ? 0
                                  : wait_start_us + (((gint64) wait_before_kill_msec) * 1000L);

        while (TRUE) {
            ret = waitpid(pid, &status, WNOHANG);
            if (ret > 0) {
                nm_log_dbg(log_domain,
                           LOG_NAME_FMT ": after sending %s, process %ld exited %s%s",
                           LOG_NAME_ARGS,
                           _kc_signal_to_string(sig),
                           (long) ret,
                           _kc_exit_to_string(buf_exit, status),
                           was_waiting ? _kc_waited_to_string(buf_wait, wait_start_us) : "");
                success = TRUE;
                goto out;
            }
            if (ret == -1) {
                errsv = errno;
                /* ECHILD means, the process is not a child/does not exist or it has SIGCHILD blocked. */
                if (errsv != ECHILD) {
                    nm_log_err(LOGD_CORE | log_domain,
                               LOG_NAME_FMT ": after sending %s, waitpid failed with %s (%d)%s",
                               LOG_NAME_ARGS,
                               _kc_signal_to_string(sig),
                               nm_strerror_native(errsv),
                               errsv,
                               was_waiting ? _kc_waited_to_string(buf_wait, wait_start_us) : "");
                    goto out;
                }
            }

            if (!wait_until)
                break;

            now = nm_utils_get_monotonic_timestamp_usec();
            if (now >= wait_until)
                break;

            if (!was_waiting) {
                nm_log_dbg(log_domain,
                           LOG_NAME_FMT ": waiting up to %ld milliseconds for process to terminate "
                                        "normally after sending %s...",
                           LOG_NAME_ARGS,
                           (long) MAX(wait_before_kill_msec, 0),
                           _kc_signal_to_string(sig));
                was_waiting = TRUE;
            }

            sleep_time = MIN(wait_until - now, sleep_duration_usec);
            if (loop_count < 20) {
                /* At the beginning we expect the process to die fast.
                 * Limit the sleep time, the limit doubles with every iteration. */
                sleep_time = MIN(sleep_time, (((guint64) 1) << loop_count) * G_USEC_PER_SEC / 2000);
                loop_count++;
            }
            g_usleep(sleep_time);
        }

        /* send SIGKILL, if called with @wait_before_kill_msec > 0 */
        if (wait_until) {
            nm_log_dbg(log_domain, LOG_NAME_FMT ": sending SIGKILL...", LOG_NAME_ARGS);

            send_kill = TRUE;
            if (kill(pid, SIGKILL) != 0) {
                errsv = errno;
                /* ESRCH means, process does not exist or is already a zombie. */
                if (errsv != ESRCH) {
                    nm_log_err(LOGD_CORE | log_domain,
                               LOG_NAME_FMT ": failed to send SIGKILL (after sending %s), %s (%d)",
                               LOG_NAME_ARGS,
                               _kc_signal_to_string(sig),
                               nm_strerror_native(errsv),
                               errsv);
                    goto out;
                }
            }
        }
    }

    if (!was_waiting) {
        nm_log_dbg(log_domain,
                   LOG_NAME_FMT ": waiting for process to terminate after sending %s%s...",
                   LOG_NAME_ARGS,
                   _kc_signal_to_string(sig),
                   send_kill ? " and SIGKILL" : "");
    }

    /* block until the child terminates. */
    while ((ret = waitpid(pid, &status, 0)) <= 0) {
        errsv = errno;

        if (errsv != EINTR) {
            nm_log_err(LOGD_CORE | log_domain,
                       LOG_NAME_FMT ": after sending %s%s, waitpid failed with %s (%d)%s",
                       LOG_NAME_ARGS,
                       _kc_signal_to_string(sig),
                       send_kill ? " and SIGKILL" : "",
                       nm_strerror_native(errsv),
                       errsv,
                       _kc_waited_to_string(buf_wait, wait_start_us));
            goto out;
        }
    }

    nm_log_dbg(log_domain,
               LOG_NAME_FMT ": after sending %s%s, process %ld exited %s%s",
               LOG_NAME_ARGS,
               _kc_signal_to_string(sig),
               send_kill ? " and SIGKILL" : "",
               (long) ret,
               _kc_exit_to_string(buf_exit, status),
               _kc_waited_to_string(buf_wait, wait_start_us));
    success = TRUE;
out:
    if (child_status)
        *child_status = success ? status : -1;
    errno = success ? 0 : errsv;
    return success;
}


nm_utils_validate_plugin(const char *path, struct stat *st, GError **error)
{
    g_return_val_if_fail(path, FALSE);
    g_return_val_if_fail(st, FALSE);
    g_return_val_if_fail(!error || !*error, FALSE);

    if (!S_ISREG(st->st_mode)) {
        g_set_error_literal(error, NM_UTILS_ERROR, NM_UTILS_ERROR_UNKNOWN, "not a regular file");
        return FALSE;
    }

    if (st->st_uid != 0) {
        g_set_error_literal(error,
                            NM_UTILS_ERROR,
                            NM_UTILS_ERROR_UNKNOWN,
                            "file has invalid owner (should be root)");
        return FALSE;
    }

    if (st->st_mode & (S_IWGRP | S_IWOTH | S_ISUID)) {
        g_set_error_literal(error,
                            NM_UTILS_ERROR,
                            NM_UTILS_ERROR_UNKNOWN,
                            "file has invalid permissions");
        return FALSE;
    }

    return TRUE;
}


_set_stable_privacy(NMUtilsStableType stable_type,
                    struct in6_addr * addr,
                    const char *      ifname,
                    const char *      network_id,
                    guint32           dad_counter,
                    const guint8 *    host_id,
                    gsize             host_id_len,
                    GError **         error)
{
    nm_auto_free_checksum GChecksum *sum = NULL;
    guint8                           digest[NM_UTILS_CHECKSUM_LENGTH_SHA256];
    guint32                          tmp[2];

    nm_assert(host_id_len);
    nm_assert(network_id);

    sum = g_checksum_new(G_CHECKSUM_SHA256);

    host_id_len = MIN(host_id_len, G_MAXUINT32);

    if (stable_type != NM_UTILS_STABLE_TYPE_UUID) {
        guint8 stable_type_uint8;

        nm_assert(stable_type < (NMUtilsStableType) 255);
        stable_type_uint8 = (guint8) stable_type;

        /* Preferably, we would always like to include the stable-type,
         * but for backward compatibility reasons, we cannot for UUID.
         *
         * That is no real problem and it is still impossible to
         * force a collision here, because of how the remaining
         * fields are hashed. That is, as we also hash @host_id_len
         * and the terminating '\0' of @network_id, it is unambiguously
         * possible to revert the process and deduce the @stable_type.
         */
        g_checksum_update(sum, &stable_type_uint8, sizeof(stable_type_uint8));
    }

    g_checksum_update(sum, addr->s6_addr, 8);
    g_checksum_update(sum, (const guchar *) ifname, strlen(ifname) + 1);
    g_checksum_update(sum, (const guchar *) network_id, strlen(network_id) + 1);
    tmp[0] = htonl(dad_counter);
    tmp[1] = htonl(host_id_len);
    g_checksum_update(sum, (const guchar *) tmp, sizeof(tmp));
    g_checksum_update(sum, (const guchar *) host_id, host_id_len);
    nm_utils_checksum_get_digest(sum, digest);

    while (_is_reserved_ipv6_iid(digest)) {
        g_checksum_reset(sum);
        tmp[0] = htonl(++dad_counter);
        g_checksum_update(sum, digest, sizeof(digest));
        g_checksum_update(sum, (const guchar *) &tmp[0], sizeof(tmp[0]));
        nm_utils_checksum_get_digest(sum, digest);
    }

    memcpy(addr->s6_addr + 8, &digest[0], 8);
    return TRUE;
}


void *zmalloc_usable(size_t size, size_t *usable) {
    void *ptr = malloc(size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = *usable = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}


static void gfar_free_tx_queues(struct gfar_private *priv)
{
	int i;

	for (i = 0; i < priv->num_tx_queues; i++)
		kfree(priv->tx_queue[i]);
}


static void gfar_free_irq(struct gfar_private *priv)
{
	int i;

	/* Free the IRQs */
	if (priv->device_flags & FSL_GIANFAR_DEV_HAS_MULTI_INTR) {
		for (i = 0; i < priv->num_grps; i++)
			free_grp_irqs(&priv->gfargrp[i]);
	} else {
		for (i = 0; i < priv->num_grps; i++)
			free_irq(gfar_irq(&priv->gfargrp[i], TX)->irq,
				 &priv->gfargrp[i]);
	}
}


static int gfar_poll_tx(struct napi_struct *napi, int budget)
{
	struct gfar_priv_grp *gfargrp =
		container_of(napi, struct gfar_priv_grp, napi_tx);
	struct gfar_private *priv = gfargrp->priv;
	struct gfar __iomem *regs = gfargrp->regs;
	struct gfar_priv_tx_q *tx_queue = NULL;
	int has_tx_work = 0;
	int i;

	/* Clear IEVENT, so interrupts aren't called again
	 * because of the packets that have already arrived
	 */
	gfar_write(&regs->ievent, IEVENT_TX_MASK);

	for_each_set_bit(i, &gfargrp->tx_bit_map, priv->num_tx_queues) {
		tx_queue = priv->tx_queue[i];
		/* run Tx cleanup to completion */
		if (tx_queue->tx_skbuff[tx_queue->skb_dirtytx]) {
			gfar_clean_tx_ring(tx_queue);
			has_tx_work = 1;
		}
	}

	if (!has_tx_work) {
		u32 imask;
		napi_complete(napi);

		spin_lock_irq(&gfargrp->grplock);
		imask = gfar_read(&regs->imask);
		imask |= IMASK_TX_DEFAULT;
		gfar_write(&regs->imask, imask);
		spin_unlock_irq(&gfargrp->grplock);
	}

	return 0;
}


static int gfar_alloc_rx_queues(struct gfar_private *priv)
{
	int i;

	for (i = 0; i < priv->num_rx_queues; i++) {
		priv->rx_queue[i] = kzalloc(sizeof(struct gfar_priv_rx_q),
					    GFP_KERNEL);
		if (!priv->rx_queue[i])
			return -ENOMEM;

		priv->rx_queue[i]->qindex = i;
		priv->rx_queue[i]->ndev = priv->ndev;
	}
	return 0;
}


static inline struct txfcb *gfar_add_fcb(struct sk_buff *skb)
{
	struct txfcb *fcb = skb_push(skb, GMAC_FCB_LEN);

	memset(fcb, 0, GMAC_FCB_LEN);

	return fcb;
}


static inline struct txbd8 *skip_txbd(struct txbd8 *bdp, int stride,
				      struct txbd8 *base, int ring_size)
{
	struct txbd8 *new_bd = bdp + stride;

	return (new_bd >= (base + ring_size)) ? (new_bd - ring_size) : new_bd;
}


static u32 cluster_entry_per_class(struct gfar_private *priv, u32 rqfar,
				   u32 class)
{
	u32 rqfpr = FPR_FILER_MASK;
	u32 rqfcr = 0x0;

	rqfar--;
	rqfcr = RQFCR_CLE | RQFCR_PID_MASK | RQFCR_CMP_EXACT;
	priv->ftp_rqfpr[rqfar] = rqfpr;
	priv->ftp_rqfcr[rqfar] = rqfcr;
	gfar_write_filer(priv, rqfar, rqfcr, rqfpr);

	rqfar--;
	rqfcr = RQFCR_CMP_NOMATCH;
	priv->ftp_rqfpr[rqfar] = rqfpr;
	priv->ftp_rqfcr[rqfar] = rqfcr;
	gfar_write_filer(priv, rqfar, rqfcr, rqfpr);

	rqfar--;
	rqfcr = RQFCR_CMP_EXACT | RQFCR_PID_PARSE | RQFCR_CLE | RQFCR_AND;
	rqfpr = class;
	priv->ftp_rqfcr[rqfar] = rqfcr;
	priv->ftp_rqfpr[rqfar] = rqfpr;
	gfar_write_filer(priv, rqfar, rqfcr, rqfpr);

	rqfar--;
	rqfcr = RQFCR_CMP_EXACT | RQFCR_PID_MASK | RQFCR_AND;
	rqfpr = class;
	priv->ftp_rqfcr[rqfar] = rqfcr;
	priv->ftp_rqfpr[rqfar] = rqfpr;
	gfar_write_filer(priv, rqfar, rqfcr, rqfpr);

	return rqfar;
}


static void gfar_init_addr_hash_table(struct gfar_private *priv)
{
	struct gfar __iomem *regs = priv->gfargrp[0].regs;

	if (priv->device_flags & FSL_GIANFAR_DEV_HAS_EXTENDED_HASH) {
		priv->extended_hash = 1;
		priv->hash_width = 9;

		priv->hash_regs[0] = &regs->igaddr0;
		priv->hash_regs[1] = &regs->igaddr1;
		priv->hash_regs[2] = &regs->igaddr2;
		priv->hash_regs[3] = &regs->igaddr3;
		priv->hash_regs[4] = &regs->igaddr4;
		priv->hash_regs[5] = &regs->igaddr5;
		priv->hash_regs[6] = &regs->igaddr6;
		priv->hash_regs[7] = &regs->igaddr7;
		priv->hash_regs[8] = &regs->gaddr0;
		priv->hash_regs[9] = &regs->gaddr1;
		priv->hash_regs[10] = &regs->gaddr2;
		priv->hash_regs[11] = &regs->gaddr3;
		priv->hash_regs[12] = &regs->gaddr4;
		priv->hash_regs[13] = &regs->gaddr5;
		priv->hash_regs[14] = &regs->gaddr6;
		priv->hash_regs[15] = &regs->gaddr7;

	} else {
		priv->extended_hash = 0;
		priv->hash_width = 8;

		priv->hash_regs[0] = &regs->gaddr0;
		priv->hash_regs[1] = &regs->gaddr1;
		priv->hash_regs[2] = &regs->gaddr2;
		priv->hash_regs[3] = &regs->gaddr3;
		priv->hash_regs[4] = &regs->gaddr4;
		priv->hash_regs[5] = &regs->gaddr5;
		priv->hash_regs[6] = &regs->gaddr6;
		priv->hash_regs[7] = &regs->gaddr7;
	}
}


static void gfar_free_rx_queues(struct gfar_private *priv)
{
	int i;

	for (i = 0; i < priv->num_rx_queues; i++)
		kfree(priv->rx_queue[i]);
}


static int gfar_remove(struct platform_device *ofdev)
{
	struct gfar_private *priv = platform_get_drvdata(ofdev);
	struct device_node *np = ofdev->dev.of_node;

	of_node_put(priv->phy_node);
	of_node_put(priv->tbi_node);

	unregister_netdev(priv->ndev);

	if (of_phy_is_fixed_link(np))
		of_phy_deregister_fixed_link(np);

	unmap_group_regs(priv);
	gfar_free_rx_queues(priv);
	gfar_free_tx_queues(priv);
	free_gfar_dev(priv);

	return 0;
}


static void disable_napi(struct gfar_private *priv)
{
	int i;

	for (i = 0; i < priv->num_grps; i++) {
		napi_disable(&priv->gfargrp[i].napi_rx);
		napi_disable(&priv->gfargrp[i].napi_tx);
	}
}


static inline void gfar_tx_checksum(struct sk_buff *skb, struct txfcb *fcb,
				    int fcb_length)
{
	/* If we're here, it's a IP packet with a TCP or UDP
	 * payload.  We set it to checksum, using a pseudo-header
	 * we provide
	 */
	u8 flags = TXFCB_DEFAULT;

	/* Tell the controller what the protocol is
	 * And provide the already calculated phcs
	 */
	if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
		flags |= TXFCB_UDP;
		fcb->phcs = (__force __be16)(udp_hdr(skb)->check);
	} else
		fcb->phcs = (__force __be16)(tcp_hdr(skb)->check);

	/* l3os is the distance between the start of the
	 * frame (skb->data) and the start of the IP hdr.
	 * l4os is the distance between the start of the
	 * l3 hdr and the l4 hdr
	 */
	fcb->l3os = (u8)(skb_network_offset(skb) - fcb_length);
	fcb->l4os = skb_network_header_len(skb);

	fcb->flags = flags;
}


static ssize_t usbip_sockfd_store(struct device *dev, struct device_attribute *attr,
			    const char *buf, size_t count)
{
	struct stub_device *sdev = dev_get_drvdata(dev);
	int sockfd = 0;
	struct socket *socket;
	int rv;
	struct task_struct *tcp_rx = NULL;
	struct task_struct *tcp_tx = NULL;

	if (!sdev) {
		dev_err(dev, "sdev is null\n");
		return -ENODEV;
	}

	rv = sscanf(buf, "%d", &sockfd);
	if (rv != 1)
		return -EINVAL;

	if (sockfd != -1) {
		int err;

		dev_info(dev, "stub up\n");

		spin_lock_irq(&sdev->ud.lock);

		if (sdev->ud.status != SDEV_ST_AVAILABLE) {
			dev_err(dev, "not ready\n");
			goto err;
		}

		socket = sockfd_lookup(sockfd, &err);
		if (!socket) {
			dev_err(dev, "failed to lookup sock");
			goto err;
		}

		if (socket->type != SOCK_STREAM) {
			dev_err(dev, "Expecting SOCK_STREAM - found %d",
				socket->type);
			goto sock_err;
		}

		/* unlock and create threads and get tasks */
		spin_unlock_irq(&sdev->ud.lock);
		tcp_rx = kthread_create(stub_rx_loop, &sdev->ud, "stub_rx");
		if (IS_ERR(tcp_rx)) {
			sockfd_put(socket);
			return -EINVAL;
		}
		tcp_tx = kthread_create(stub_tx_loop, &sdev->ud, "stub_tx");
		if (IS_ERR(tcp_tx)) {
			kthread_stop(tcp_rx);
			sockfd_put(socket);
			return -EINVAL;
		}

		/* get task structs now */
		get_task_struct(tcp_rx);
		get_task_struct(tcp_tx);

		/* lock and update sdev->ud state */
		spin_lock_irq(&sdev->ud.lock);
		sdev->ud.tcp_socket = socket;
		sdev->ud.sockfd = sockfd;
		sdev->ud.tcp_rx = tcp_rx;
		sdev->ud.tcp_tx = tcp_tx;
		sdev->ud.status = SDEV_ST_USED;
		spin_unlock_irq(&sdev->ud.lock);

		wake_up_process(sdev->ud.tcp_rx);
		wake_up_process(sdev->ud.tcp_tx);

	} else {
		dev_info(dev, "stub down\n");

		spin_lock_irq(&sdev->ud.lock);
		if (sdev->ud.status != SDEV_ST_USED)
			goto err;

		spin_unlock_irq(&sdev->ud.lock);

		usbip_event_add(&sdev->ud, SDEV_EVENT_DOWN);
	}

	return count;

sock_err:
	sockfd_put(socket);
err:
	spin_unlock_irq(&sdev->ud.lock);
	return -EINVAL;
}


static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)
{
	struct vhost_dev *dev = &v->vdev;
	struct vdpa_device *vdpa = v->vdpa;
	const struct vdpa_config_ops *ops = vdpa->config;

	vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);

	if (ops->dma_map) {
		ops->dma_unmap(vdpa, iova, size);
	} else if (ops->set_map) {
		if (!v->in_batch)
			ops->set_map(vdpa, dev->iotlb);
	} else {
		iommu_unmap(v->domain, iova, size);
	}
}


static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
{
	struct vdpa_callback cb;
	int fd;
	struct eventfd_ctx *ctx;

	cb.callback = vhost_vdpa_config_cb;
	cb.private = v->vdpa;
	if (copy_from_user(&fd, argp, sizeof(fd)))
		return  -EFAULT;

	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
	swap(ctx, v->config_ctx);

	if (!IS_ERR_OR_NULL(ctx))
		eventfd_ctx_put(ctx);

	if (IS_ERR(v->config_ctx))
		return PTR_ERR(v->config_ctx);

	v->vdpa->config->set_config_cb(v->vdpa, &cb);

	return 0;
}


static void vhost_vdpa_release_dev(struct device *device)
{
	struct vhost_vdpa *v =
	       container_of(device, struct vhost_vdpa, dev);

	ida_simple_remove(&vhost_vdpa_ida, v->minor);
	kfree(v->vqs);
	kfree(v);
}


static int perm_to_iommu_flags(u32 perm)
{
	int flags = 0;

	switch (perm) {
	case VHOST_ACCESS_WO:
		flags |= IOMMU_WRITE;
		break;
	case VHOST_ACCESS_RO:
		flags |= IOMMU_READ;
		break;
	case VHOST_ACCESS_RW:
		flags |= (IOMMU_WRITE | IOMMU_READ);
		break;
	default:
		WARN(1, "invalidate vhost IOTLB permission\n");
		break;
	}

	return flags | IOMMU_CACHE;
}


int tipc_node_get_mtu(struct net *net, u32 addr, u32 sel, bool connected)
{
	struct tipc_node *n;
	int bearer_id;
	unsigned int mtu = MAX_MSG_SIZE;

	n = tipc_node_find(net, addr);
	if (unlikely(!n))
		return mtu;

	/* Allow MAX_MSG_SIZE when building connection oriented message
	 * if they are in the same core network
	 */
	if (n->peer_net && connected) {
		tipc_node_put(n);
		return mtu;
	}

	bearer_id = n->active_links[sel & 1];
	if (likely(bearer_id != INVALID_BEARER_ID))
		mtu = n->links[bearer_id].mtu;
	tipc_node_put(n);
	return mtu;
}


static void  tipc_node_clear_links(struct tipc_node *node)
{
	int i;

	for (i = 0; i < MAX_BEARERS; i++) {
		struct tipc_link_entry *le = &node->links[i];

		if (le->link) {
			kfree(le->link);
			le->link = NULL;
			node->link_cnt--;
		}
	}
}


static void tipc_node_kref_release(struct kref *kref)
{
	struct tipc_node *n = container_of(kref, struct tipc_node, kref);

	kfree(n->bc_entry.link);
	call_rcu(&n->rcu, tipc_node_free);
}


static struct tipc_node *tipc_node_find_by_name(struct net *net,
						const char *link_name,
						unsigned int *bearer_id)
{
	struct tipc_net *tn = net_generic(net, tipc_net_id);
	struct tipc_link *l;
	struct tipc_node *n;
	struct tipc_node *found_node = NULL;
	int i;

	*bearer_id = 0;
	rcu_read_lock();
	list_for_each_entry_rcu(n, &tn->node_list, list) {
		tipc_node_read_lock(n);
		for (i = 0; i < MAX_BEARERS; i++) {
			l = n->links[i].link;
			if (l && !strcmp(tipc_link_name(l), link_name)) {
				*bearer_id = i;
				found_node = n;
				break;
			}
		}
		tipc_node_read_unlock(n);
		if (found_node)
			break;
	}
	rcu_read_unlock();

	return found_node;
}


int tipc_node_add_conn(struct net *net, u32 dnode, u32 port, u32 peer_port)
{
	struct tipc_node *node;
	struct tipc_sock_conn *conn;
	int err = 0;

	if (in_own_node(net, dnode))
		return 0;

	node = tipc_node_find(net, dnode);
	if (!node) {
		pr_warn("Connecting sock to node 0x%x failed\n", dnode);
		return -EHOSTUNREACH;
	}
	conn = kmalloc(sizeof(*conn), GFP_ATOMIC);
	if (!conn) {
		err = -EHOSTUNREACH;
		goto exit;
	}
	conn->peer_node = dnode;
	conn->port = port;
	conn->peer_port = peer_port;

	tipc_node_write_lock(node);
	list_add_tail(&conn->list, &node->conn_sks);
	tipc_node_write_unlock(node);
exit:
	tipc_node_put(node);
	return err;
}


static void tipc_node_link_failover(struct tipc_node *n, struct tipc_link *l,
				    struct tipc_link *tnl,
				    struct sk_buff_head *xmitq)
{
	/* Avoid to be "self-failover" that can never end */
	if (!tipc_link_is_up(tnl))
		return;

	/* Don't rush, failure link may be in the process of resetting */
	if (l && !tipc_link_is_reset(l))
		return;

	tipc_link_fsm_evt(tnl, LINK_SYNCH_END_EVT);
	tipc_node_fsm_evt(n, NODE_SYNCH_END_EVT);

	n->sync_point = tipc_link_rcv_nxt(tnl) + (U16_MAX / 2 - 1);
	tipc_link_failover_prepare(l, tnl, xmitq);

	if (l)
		tipc_link_fsm_evt(l, LINK_FAILOVER_BEGIN_EVT);
	tipc_node_fsm_evt(n, NODE_FAILOVER_BEGIN_EVT);
}


int tipc_node_dump(struct tipc_node *n, bool more, char *buf)
{
	int i = 0;
	size_t sz = (more) ? NODE_LMAX : NODE_LMIN;

	if (!n) {
		i += scnprintf(buf, sz, "node data: (null)\n");
		return i;
	}

	i += scnprintf(buf, sz, "node data: %x", n->addr);
	i += scnprintf(buf + i, sz - i, " %x", n->state);
	i += scnprintf(buf + i, sz - i, " %d", n->active_links[0]);
	i += scnprintf(buf + i, sz - i, " %d", n->active_links[1]);
	i += scnprintf(buf + i, sz - i, " %x", n->action_flags);
	i += scnprintf(buf + i, sz - i, " %u", n->failover_sent);
	i += scnprintf(buf + i, sz - i, " %u", n->sync_point);
	i += scnprintf(buf + i, sz - i, " %d", n->link_cnt);
	i += scnprintf(buf + i, sz - i, " %u", n->working_links);
	i += scnprintf(buf + i, sz - i, " %x", n->capabilities);
	i += scnprintf(buf + i, sz - i, " %lu\n", n->keepalive_intv);

	if (!more)
		return i;

	i += scnprintf(buf + i, sz - i, "link_entry[0]:\n");
	i += scnprintf(buf + i, sz - i, " mtu: %u\n", n->links[0].mtu);
	i += scnprintf(buf + i, sz - i, " media: ");
	i += tipc_media_addr_printf(buf + i, sz - i, &n->links[0].maddr);
	i += scnprintf(buf + i, sz - i, "\n");
	i += tipc_link_dump(n->links[0].link, TIPC_DUMP_NONE, buf + i);
	i += scnprintf(buf + i, sz - i, " inputq: ");
	i += tipc_list_dump(&n->links[0].inputq, false, buf + i);

	i += scnprintf(buf + i, sz - i, "link_entry[1]:\n");
	i += scnprintf(buf + i, sz - i, " mtu: %u\n", n->links[1].mtu);
	i += scnprintf(buf + i, sz - i, " media: ");
	i += tipc_media_addr_printf(buf + i, sz - i, &n->links[1].maddr);
	i += scnprintf(buf + i, sz - i, "\n");
	i += tipc_link_dump(n->links[1].link, TIPC_DUMP_NONE, buf + i);
	i += scnprintf(buf + i, sz - i, " inputq: ");
	i += tipc_list_dump(&n->links[1].inputq, false, buf + i);

	i += scnprintf(buf + i, sz - i, "bclink:\n ");
	i += tipc_link_dump(n->bc_entry.link, TIPC_DUMP_NONE, buf + i);

	return i;
}


static int __tipc_nl_add_node_links(struct net *net, struct tipc_nl_msg *msg,
				    struct tipc_node *node, u32 *prev_link,
				    bool bc_link)
{
	u32 i;
	int err;

	for (i = *prev_link; i < MAX_BEARERS; i++) {
		*prev_link = i;

		if (!node->links[i].link)
			continue;

		err = __tipc_nl_add_link(net, msg,
					 node->links[i].link, NLM_F_MULTI);
		if (err)
			return err;
	}

	if (bc_link) {
		*prev_link = i;
		err = tipc_nl_add_bc_link(net, msg, node->bc_entry.link);
		if (err)
			return err;
	}

	*prev_link = 0;

	return 0;
}


static void tipc_node_delete(struct tipc_node *node)
{
	trace_tipc_node_delete(node, true, " ");
	tipc_node_delete_from_list(node);

	del_timer_sync(&node->timer);
	tipc_node_put(node);
}


static int qrtr_port_assign(struct qrtr_sock *ipc, int *port)
{
	u32 min_port;
	int rc;

	mutex_lock(&qrtr_port_lock);
	if (!*port) {
		min_port = QRTR_MIN_EPH_SOCKET;
		rc = idr_alloc_u32(&qrtr_ports, ipc, &min_port, QRTR_MAX_EPH_SOCKET, GFP_ATOMIC);
		if (!rc)
			*port = min_port;
	} else if (*port < QRTR_MIN_EPH_SOCKET && !capable(CAP_NET_ADMIN)) {
		rc = -EACCES;
	} else if (*port == QRTR_PORT_CTRL) {
		min_port = 0;
		rc = idr_alloc_u32(&qrtr_ports, ipc, &min_port, 0, GFP_ATOMIC);
	} else {
		min_port = *port;
		rc = idr_alloc_u32(&qrtr_ports, ipc, &min_port, *port, GFP_ATOMIC);
		if (!rc)
			*port = min_port;
	}
	mutex_unlock(&qrtr_port_lock);

	if (rc == -ENOSPC)
		return -EADDRINUSE;
	else if (rc < 0)
		return rc;

	sock_hold(&ipc->sk);

	return 0;
}


int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
{
	const char *end = src + size;
	const char *orig_src = src;

	memset(dst, 0, size);
	/* Copy all isalnum(), '_' and '.' chars. */
	while (src < end && *src) {
		if (!isalnum(*src) &&
		    *src != '_' && *src != '.')
			return -EINVAL;
		*dst++ = *src++;
	}

	/* No '\0' found in "size" number of bytes */
	if (src == end)
		return -EINVAL;

	return src - orig_src;
}


static int bpf_obj_get(const union bpf_attr *attr)
{
	if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
	    attr->file_flags & ~BPF_OBJ_FLAG_MASK)
		return -EINVAL;

	return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
				attr->file_flags);
}


static int bpf_map_alloc_id(struct bpf_map *map)
{
	int id;

	idr_preload(GFP_KERNEL);
	spin_lock_bh(&map_idr_lock);
	id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
	if (id > 0)
		map->id = id;
	spin_unlock_bh(&map_idr_lock);
	idr_preload_end();

	if (WARN_ON_ONCE(!id))
		return -ENOSPC;

	return id > 0 ? 0 : id;
}


static void bpf_raw_tp_link_release(struct bpf_link *link)
{
	struct bpf_raw_tp_link *raw_tp =
		container_of(link, struct bpf_raw_tp_link, link);

	bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
	bpf_put_raw_tracepoint(raw_tp->btp);
}


struct bpf_map *bpf_map_get_with_uref(u32 ufd)
{
	struct fd f = fdget(ufd);
	struct bpf_map *map;

	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return map;

	bpf_map_inc_with_uref(map);
	fdput(f);

	return map;
}


struct bpf_prog *bpf_prog_by_id(u32 id)
{
	struct bpf_prog *prog;

	if (!id)
		return ERR_PTR(-ENOENT);

	spin_lock_bh(&prog_idr_lock);
	prog = idr_find(&prog_idr, id);
	if (prog)
		prog = bpf_prog_inc_not_zero(prog);
	else
		prog = ERR_PTR(-ENOENT);
	spin_unlock_bh(&prog_idr_lock);
	return prog;
}


static void bpf_prog_get_stats(const struct bpf_prog *prog,
			       struct bpf_prog_stats *stats)
{
	u64 nsecs = 0, cnt = 0, misses = 0;
	int cpu;

	for_each_possible_cpu(cpu) {
		const struct bpf_prog_stats *st;
		unsigned int start;
		u64 tnsecs, tcnt, tmisses;

		st = per_cpu_ptr(prog->stats, cpu);
		do {
			start = u64_stats_fetch_begin_irq(&st->syncp);
			tnsecs = st->nsecs;
			tcnt = st->cnt;
			tmisses = st->misses;
		} while (u64_stats_fetch_retry_irq(&st->syncp, start));
		nsecs += tnsecs;
		cnt += tcnt;
		misses += tmisses;
	}
	stats->nsecs = nsecs;
	stats->cnt = cnt;
	stats->misses = misses;
}


static void *__bpf_copy_key(void __user *ukey, u64 key_size)
{
	if (key_size)
		return memdup_user(ukey, key_size);

	if (ukey)
		return ERR_PTR(-EINVAL);

	return NULL;
}