static void __init resiliency_test(void)
{
	u8 *p;
	int type = KMALLOC_NORMAL;

	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);

	pr_err("SLUB resiliency testing\n");
	pr_err("-----------------------\n");
	pr_err("A. Corruption after allocation\n");

	p = kzalloc(16, GFP_KERNEL);
	p[16] = 0x12;
	pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
	       p + 16);

	validate_slab_cache(kmalloc_caches[type][4]);

	/* Hmmm... The next two are dangerous */
	p = kzalloc(32, GFP_KERNEL);
	p[32 + sizeof(void *)] = 0x34;
	pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");

	validate_slab_cache(kmalloc_caches[type][5]);
	p = kzalloc(64, GFP_KERNEL);
	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
	*p = 0x56;
	pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");
	validate_slab_cache(kmalloc_caches[type][6]);

	pr_err("\nB. Corruption after free\n");
	p = kzalloc(128, GFP_KERNEL);
	kfree(p);
	*p = 0x78;
	pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[type][7]);

	p = kzalloc(256, GFP_KERNEL);
	kfree(p);
	p[50] = 0x9a;
	pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[type][8]);

	p = kzalloc(512, GFP_KERNEL);
	kfree(p);
	p[512] = 0xab;
	pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[type][9]);
}


static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
			u8 *object, char *what,
			u8 *start, unsigned int value, unsigned int bytes)
{
	u8 *fault;
	u8 *end;
	u8 *addr = page_address(page);

	metadata_access_enable();
	fault = memchr_inv(start, value, bytes);
	metadata_access_disable();
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
	pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
					fault, end - 1, fault - addr,
					fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
}


static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }


static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
				unsigned long *pos, void *start,
				unsigned long page_limit,
				unsigned long freelist_count)
{
	unsigned int idx;

	/*
	 * If the target page allocation failed, the number of objects on the
	 * page might be smaller than the usual size defined by the cache.
	 */
	do {
		idx = s->random_seq[*pos];
		*pos += 1;
		if (*pos >= freelist_count)
			*pos = 0;
	} while (unlikely(idx >= page_limit));

	return (char *)start + idx;
}


static void init_object(struct kmem_cache *s, void *object, u8 val)
{
	u8 *p = object;

	if (s->flags & SLAB_RED_ZONE)
		memset(p - s->red_left_pad, val, s->red_left_pad);

	if (s->flags & __OBJECT_POISON) {
		memset(p, POISON_FREE, s->object_size - 1);
		p[s->object_size - 1] = POISON_END;
	}

	if (s->flags & SLAB_RED_ZONE)
		memset(p + s->object_size, val, s->inuse - s->object_size);
}


void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);

	trace_kmem_cache_alloc_node(_RET_IP_, ret,
				    s->object_size, s->size, gfpflags, node);

	return ret;
}


static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
{
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

	if (c->page)
		flush_slab(s, c);

	unfreeze_partials(s, c);
}


static inline int node_match(struct page *page, int node)
{
#ifdef CONFIG_NUMA
	if (node != NUMA_NO_NODE && page_to_nid(page) != node)
		return 0;
#endif
	return 1;
}


static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x)
{
	kmemleak_free_recursive(x, s->flags);

	/*
	 * Trouble is that we may no longer disable interrupts in the fast path
	 * So in order to make the debug calls that expect irqs to be
	 * disabled we need to disable interrupts temporarily.
	 */
#ifdef CONFIG_LOCKDEP
	{
		unsigned long flags;

		local_irq_save(flags);
		debug_check_no_locks_freed(x, s->object_size);
		local_irq_restore(flags);
	}
#endif
	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->object_size);

	/* KASAN might put x into memory quarantine, delaying its reuse */
	return kasan_slab_free(s, x, _RET_IP_);
}


static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *p;
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPTION
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif

	p = ___slab_alloc(s, gfpflags, node, addr, c);
	local_irq_restore(flags);
	return p;
}


static inline void init_freelist_randomization(void) { }


static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *freelist;
	struct page *page;

	page = c->page;
	if (!page)
		goto new_slab;
redo:

	if (unlikely(!node_match(page, node))) {
		int searchnode = node;

		if (node != NUMA_NO_NODE && !node_present_pages(node))
			searchnode = node_to_mem_node(node);

		if (unlikely(!node_match(page, searchnode))) {
			stat(s, ALLOC_NODE_MISMATCH);
			deactivate_slab(s, page, c->freelist, c);
			goto new_slab;
		}
	}

	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
	if (unlikely(!pfmemalloc_match(page, gfpflags))) {
		deactivate_slab(s, page, c->freelist, c);
		goto new_slab;
	}

	/* must check again c->freelist in case of cpu migration or IRQ */
	freelist = c->freelist;
	if (freelist)
		goto load_freelist;

	freelist = get_freelist(s, page);

	if (!freelist) {
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
		goto new_slab;
	}

	stat(s, ALLOC_REFILL);

load_freelist:
	/*
	 * freelist is pointing to the list of objects to be used.
	 * page is pointing to the page from which the objects are obtained.
	 * That page must be frozen for per cpu allocations to work.
	 */
	VM_BUG_ON(!c->page->frozen);
	c->freelist = get_freepointer(s, freelist);
	c->tid = next_tid(c->tid);
	return freelist;

new_slab:

	if (slub_percpu_partial(c)) {
		page = c->page = slub_percpu_partial(c);
		slub_set_percpu_partial(c, page);
		stat(s, CPU_PARTIAL_ALLOC);
		goto redo;
	}

	freelist = new_slab_objects(s, gfpflags, node, &c);

	if (unlikely(!freelist)) {
		slab_out_of_memory(s, gfpflags, node);
		return NULL;
	}

	page = c->page;
	if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
		goto load_freelist;

	/* Only entered in the debug case */
	if (kmem_cache_debug(s) &&
			!alloc_debug_processing(s, page, freelist, addr))
		goto new_slab;	/* Slab failed checks. Next slab needed */

	deactivate_slab(s, page, get_freepointer(s, freelist), c);
	return freelist;
}


static ssize_t sanity_checks_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_CONSISTENCY_CHECKS;
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
		s->flags |= SLAB_CONSISTENCY_CHECKS;
	}
	return length;
}


static void flush_all(struct kmem_cache *s)
{
	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1);
}


static long madvise_willneed(struct vm_area_struct *vma,
			     struct vm_area_struct **prev,
			     unsigned long start, unsigned long end)
{
	struct file *file = vma->vm_file;
	loff_t offset;

	*prev = vma;
#ifdef CONFIG_SWAP
	if (!file) {
		walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
		lru_add_drain(); /* Push any new pages onto the LRU now */
		return 0;
	}

	if (shmem_mapping(file->f_mapping)) {
		force_shm_swapin_readahead(vma, start, end,
					file->f_mapping);
		return 0;
	}
#else
	if (!file)
		return -EBADF;
#endif

	if (IS_DAX(file_inode(file))) {
		/* no bad return value, but ignore advice */
		return 0;
	}

	/*
	 * Filesystem's fadvise may need to take various locks.  We need to
	 * explicitly grab a reference because the vma (and hence the
	 * vma's reference to the file) can go away as soon as we drop
	 * mmap_sem.
	 */
	*prev = NULL;	/* tell sys_madvise we drop mmap_sem */
	get_file(file);
	up_read(&current->mm->mmap_sem);
	offset = (loff_t)(start - vma->vm_start)
			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
	vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
	fput(file);
	down_read(&current->mm->mmap_sem);
	return 0;
}


static void madvise_cold_page_range(struct mmu_gather *tlb,
			     struct vm_area_struct *vma,
			     unsigned long addr, unsigned long end)
{
	struct madvise_walk_private walk_private = {
		.pageout = false,
		.tlb = tlb,
	};

	tlb_start_vma(tlb, vma);
	walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
	tlb_end_vma(tlb, vma);
}


static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			struct io_kiocb **nxt, bool force_nonblock)
{
	struct io_ring_ctx *ctx = req->ctx;
	int ret;

	switch (req->opcode) {
	case IORING_OP_NOP:
		ret = io_nop(req);
		break;
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
		if (sqe) {
			ret = io_read_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
		ret = io_read(req, nxt, force_nonblock);
		break;
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		if (sqe) {
			ret = io_write_prep(req, sqe, force_nonblock);
			if (ret < 0)
				break;
		}
		ret = io_write(req, nxt, force_nonblock);
		break;
	case IORING_OP_FSYNC:
		if (sqe) {
			ret = io_prep_fsync(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_fsync(req, nxt, force_nonblock);
		break;
	case IORING_OP_POLL_ADD:
		if (sqe) {
			ret = io_poll_add_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_poll_add(req, nxt);
		break;
	case IORING_OP_POLL_REMOVE:
		if (sqe) {
			ret = io_poll_remove_prep(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_poll_remove(req);
		break;
	case IORING_OP_SYNC_FILE_RANGE:
		if (sqe) {
			ret = io_prep_sfr(req, sqe);
			if (ret < 0)
				break;
		}
		ret = io_sync_file_range(req, nxt, force_nonblock);
		break;
	case IORING_OP_SENDMSG:
	case IORING_OP_SEND:
		if (sqe) {
			ret = io_sendmsg_prep(req, sqe);
			if (ret < 0)
				break;
		}
		if (req->opcode == IORING_OP_SENDMSG)
			ret = io_sendmsg(req, nxt, force_nonblock);
		else
			ret = io_send(req, nxt, force_nonblock);
		break;
	case IORING_OP_RECVMSG:
	case IORING_OP_RECV:
		if (sqe) {
			ret = io_recvmsg_prep(req, sqe);
			if (ret)
				break;
		}
		if (req->opcode == IORING_OP_RECVMSG)
			ret = io_recvmsg(req, nxt, force_nonblock);
		else
			ret = io_recv(req, nxt, force_nonblock);
		break;
	case IORING_OP_TIMEOUT:
		if (sqe) {
			ret = io_timeout_prep(req, sqe, false);
			if (ret)
				break;
		}
		ret = io_timeout(req);
		break;
	case IORING_OP_TIMEOUT_REMOVE:
		if (sqe) {
			ret = io_timeout_remove_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_timeout_remove(req);
		break;
	case IORING_OP_ACCEPT:
		if (sqe) {
			ret = io_accept_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_accept(req, nxt, force_nonblock);
		break;
	case IORING_OP_CONNECT:
		if (sqe) {
			ret = io_connect_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_connect(req, nxt, force_nonblock);
		break;
	case IORING_OP_ASYNC_CANCEL:
		if (sqe) {
			ret = io_async_cancel_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_async_cancel(req, nxt);
		break;
	case IORING_OP_FALLOCATE:
		if (sqe) {
			ret = io_fallocate_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fallocate(req, nxt, force_nonblock);
		break;
	case IORING_OP_OPENAT:
		if (sqe) {
			ret = io_openat_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat(req, nxt, force_nonblock);
		break;
	case IORING_OP_CLOSE:
		if (sqe) {
			ret = io_close_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_close(req, nxt, force_nonblock);
		break;
	case IORING_OP_FILES_UPDATE:
		if (sqe) {
			ret = io_files_update_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_files_update(req, force_nonblock);
		break;
	case IORING_OP_STATX:
		if (sqe) {
			ret = io_statx_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_statx(req, nxt, force_nonblock);
		break;
	case IORING_OP_FADVISE:
		if (sqe) {
			ret = io_fadvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_fadvise(req, nxt, force_nonblock);
		break;
	case IORING_OP_MADVISE:
		if (sqe) {
			ret = io_madvise_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_madvise(req, nxt, force_nonblock);
		break;
	case IORING_OP_OPENAT2:
		if (sqe) {
			ret = io_openat2_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_openat2(req, nxt, force_nonblock);
		break;
	case IORING_OP_EPOLL_CTL:
		if (sqe) {
			ret = io_epoll_ctl_prep(req, sqe);
			if (ret)
				break;
		}
		ret = io_epoll_ctl(req, nxt, force_nonblock);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	if (ret)
		return ret;

	if (ctx->flags & IORING_SETUP_IOPOLL) {
		const bool in_async = io_wq_current_is_worker();

		if (req->result == -EAGAIN)
			return -EAGAIN;

		/* workqueue context doesn't hold uring_lock, grab it now */
		if (in_async)
			mutex_lock(&ctx->uring_lock);

		io_iopoll_req_issued(req);

		if (in_async)
			mutex_unlock(&ctx->uring_lock);
	}

	return 0;
}


static bool io_queue_file_removal(struct fixed_file_data *data,
				  struct file *file)
{
	struct io_file_put *pfile, pfile_stack;
	DECLARE_COMPLETION_ONSTACK(done);

	/*
	 * If we fail allocating the struct we need for doing async reomval
	 * of this file, just punt to sync and wait for it.
	 */
	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
	if (!pfile) {
		pfile = &pfile_stack;
		pfile->done = &done;
	}

	pfile->file = file;
	llist_add(&pfile->llist, &data->put_llist);

	if (pfile == &pfile_stack) {
		if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
			percpu_ref_put(&data->refs);
			percpu_ref_switch_to_atomic(&data->refs,
							io_atomic_switch);
		}
		wait_for_completion(&done);
		flush_work(&data->ref_work);
		return false;
	}

	return true;
}


static void *io_uring_validate_mmap_request(struct file *file,
					    loff_t pgoff, size_t sz)
{
	struct io_ring_ctx *ctx = file->private_data;
	loff_t offset = pgoff << PAGE_SHIFT;
	struct page *page;
	void *ptr;

	switch (offset) {
	case IORING_OFF_SQ_RING:
	case IORING_OFF_CQ_RING:
		ptr = ctx->rings;
		break;
	case IORING_OFF_SQES:
		ptr = ctx->sq_sqes;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}

	page = virt_to_head_page(ptr);
	if (sz > page_size(page))
		return ERR_PTR(-EINVAL);

	return ptr;
}


static int io_fallocate_prep(struct io_kiocb *req,
			     const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
		return -EINVAL;

	req->sync.off = READ_ONCE(sqe->off);
	req->sync.len = READ_ONCE(sqe->addr);
	req->sync.mode = READ_ONCE(sqe->len);
	return 0;
}


static void io_cleanup_req(struct io_kiocb *req)
{
	struct io_async_ctx *io = req->io;

	switch (req->opcode) {
	case IORING_OP_READV:
	case IORING_OP_READ_FIXED:
	case IORING_OP_READ:
	case IORING_OP_WRITEV:
	case IORING_OP_WRITE_FIXED:
	case IORING_OP_WRITE:
		if (io->rw.iov != io->rw.fast_iov)
			kfree(io->rw.iov);
		break;
	case IORING_OP_SENDMSG:
	case IORING_OP_RECVMSG:
		if (io->msg.iov != io->msg.fast_iov)
			kfree(io->msg.iov);
		break;
	case IORING_OP_OPENAT:
	case IORING_OP_OPENAT2:
	case IORING_OP_STATX:
		putname(req->open.filename);
		break;
	}

	req->flags &= ~REQ_F_NEED_CLEANUP;
}


static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			bool force_nonblock)
{
	struct io_async_ctx *io;
	struct iov_iter iter;
	ssize_t ret;

	ret = io_prep_rw(req, sqe, force_nonblock);
	if (ret)
		return ret;

	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		return -EBADF;

	if (!req->io)
		return 0;

	io = req->io;
	io->rw.iov = io->rw.fast_iov;
	req->io = NULL;
	ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
	req->io = io;
	if (ret < 0)
		return ret;

	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	return 0;
}


static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
			       unsigned nr_args)
{
	struct io_uring_files_update up;

	if (!ctx->file_data)
		return -ENXIO;
	if (!nr_args)
		return -EINVAL;
	if (copy_from_user(&up, arg, sizeof(up)))
		return -EFAULT;
	if (up.resv)
		return -EINVAL;

	return __io_sqe_files_update(ctx, &up, nr_args);
}


static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
	struct io_kiocb *nxt = NULL;

	io_complete_rw_common(kiocb, res);
	io_put_req_find_next(req, &nxt);

	return nxt;
}


static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
			  struct file *ring_file, int ring_fd,
			  struct mm_struct **mm, bool async)
{
	struct io_submit_state state, *statep = NULL;
	struct io_kiocb *link = NULL;
	int i, submitted = 0;
	bool mm_fault = false;

	/* if we have a backlog and couldn't flush it all, return BUSY */
	if (test_bit(0, &ctx->sq_check_overflow)) {
		if (!list_empty(&ctx->cq_overflow_list) &&
		    !io_cqring_overflow_flush(ctx, false))
			return -EBUSY;
	}

	/* make sure SQ entry isn't read before tail */
	nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));

	if (!percpu_ref_tryget_many(&ctx->refs, nr))
		return -EAGAIN;

	if (nr > IO_PLUG_THRESHOLD) {
		io_submit_state_start(&state, nr);
		statep = &state;
	}

	ctx->ring_fd = ring_fd;
	ctx->ring_file = ring_file;

	for (i = 0; i < nr; i++) {
		const struct io_uring_sqe *sqe;
		struct io_kiocb *req;
		int err;

		req = io_get_req(ctx, statep);
		if (unlikely(!req)) {
			if (!submitted)
				submitted = -EAGAIN;
			break;
		}
		if (!io_get_sqring(ctx, req, &sqe)) {
			__io_req_do_free(req);
			break;
		}

		/* will complete beyond this point, count as submitted */
		submitted++;

		if (unlikely(req->opcode >= IORING_OP_LAST)) {
			err = -EINVAL;
fail_req:
			io_cqring_add_event(req, err);
			io_double_put_req(req);
			break;
		}

		if (io_op_defs[req->opcode].needs_mm && !*mm) {
			mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
			if (unlikely(mm_fault)) {
				err = -EFAULT;
				goto fail_req;
			}
			use_mm(ctx->sqo_mm);
			*mm = ctx->sqo_mm;
		}

		req->in_async = async;
		req->needs_fixed_file = async;
		trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
						true, async);
		if (!io_submit_sqe(req, sqe, statep, &link))
			break;
	}

	if (unlikely(submitted != nr)) {
		int ref_used = (submitted == -EAGAIN) ? 0 : submitted;

		percpu_ref_put_many(&ctx->refs, nr - ref_used);
	}
	if (link)
		io_queue_link_head(link);
	if (statep)
		io_submit_state_end(&state);

	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
	io_commit_sqring(ctx);

	return submitted;
}


static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
			unpin_user_page(imu->bvec[j].bv_page);

		if (ctx->account_mem)
			io_unaccount_mem(ctx->user, imu->nr_bvecs);
		kvfree(imu->bvec);
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}


static void io_file_ref_kill(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	complete(&data->done);
}


static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	struct io_open *ctx = &req->open;
	unsigned lookup_flags;
	struct path path;
	struct kstat stat;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
		return -EINVAL;

retry:
	/* filename_lookup() drops it, keep a reference */
	ctx->filename->refcnt++;

	ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
				NULL);
	if (ret)
		goto err;

	ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
	path_put(&path);
	if (retry_estale(ret, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
	if (!ret)
		ret = cp_statx(&stat, ctx->buffer);
err:
	putname(ctx->filename);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
}


static bool io_file_supports_async(struct file *file)
{
	umode_t mode = file_inode(file)->i_mode;

	if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
		return true;
	if (S_ISREG(mode) && file->f_op != &io_uring_fops)
		return true;

	return false;
}


static void io_poll_flush(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct llist_node *nodes;

	nodes = llist_del_all(&req->ctx->poll_llist);
	if (nodes)
		__io_poll_flush(req->ctx, nodes);
}


static void io_commit_cqring(struct io_ring_ctx *ctx)
{
	struct io_kiocb *req;

	while ((req = io_get_timeout_req(ctx)) != NULL)
		io_kill_timeout(req);

	__io_commit_cqring(ctx);

	while ((req = io_get_deferred_req(ctx)) != NULL)
		io_queue_async_work(req);
}


static void io_fsync_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	loff_t end = req->sync.off + req->sync.len;
	struct io_kiocb *nxt = NULL;
	int ret;

	if (io_req_cancelled(req))
		return;

	ret = vfs_fsync_range(req->file, req->sync.off,
				end > 0 ? end : LLONG_MAX,
				req->sync.flags & IORING_FSYNC_DATASYNC);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, &nxt);
	if (nxt)
		io_wq_assign_next(workptr, nxt);
}


static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
			       struct iov_iter *iter)
{
	struct io_ring_ctx *ctx = req->ctx;
	size_t len = req->rw.len;
	struct io_mapped_ubuf *imu;
	unsigned index, buf_index;
	size_t offset;
	u64 buf_addr;

	/* attempt to use fixed buffers without having provided iovecs */
	if (unlikely(!ctx->user_bufs))
		return -EFAULT;

	buf_index = (unsigned long) req->rw.kiocb.private;
	if (unlikely(buf_index >= ctx->nr_user_bufs))
		return -EFAULT;

	index = array_index_nospec(buf_index, ctx->nr_user_bufs);
	imu = &ctx->user_bufs[index];
	buf_addr = req->rw.addr;

	/* overflow */
	if (buf_addr + len < buf_addr)
		return -EFAULT;
	/* not inside the mapped region */
	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
		return -EFAULT;

	/*
	 * May not be a start of buffer, set size appropriately
	 * and advance us to the beginning.
	 */
	offset = buf_addr - imu->ubuf;
	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);

	if (offset) {
		/*
		 * Don't use iov_iter_advance() here, as it's really slow for
		 * using the latter parts of a big fixed buffer - it iterates
		 * over each segment manually. We can cheat a bit here, because
		 * we know that:
		 *
		 * 1) it's a BVEC iter, we set it up
		 * 2) all bvecs are PAGE_SIZE in size, except potentially the
		 *    first and last bvec
		 *
		 * So just find our index, and adjust the iterator afterwards.
		 * If the offset is within the first bvec (or the whole first
		 * bvec, just use iov_iter_advance(). This makes it easier
		 * since we can just skip the first segment, which may not
		 * be PAGE_SIZE aligned.
		 */
		const struct bio_vec *bvec = imu->bvec;

		if (offset <= bvec->bv_len) {
			iov_iter_advance(iter, offset);
		} else {
			unsigned long seg_skip;

			/* skip first vec */
			offset -= bvec->bv_len;
			seg_skip = 1 + (offset >> PAGE_SHIFT);

			iter->bvec = bvec + seg_skip;
			iter->nr_segs -= seg_skip;
			iter->count -= bvec->bv_len + offset;
			iter->iov_offset = offset & ~PAGE_MASK;
		}
	}

	return len;
}


static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
#if defined(CONFIG_NET)
	struct io_async_msghdr *kmsg = NULL;
	struct socket *sock;
	int ret;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	sock = sock_from_file(req->file, &ret);
	if (sock) {
		struct io_async_ctx io;
		struct sockaddr_storage addr;
		unsigned flags;

		if (req->io) {
			kmsg = &req->io->msg;
			kmsg->msg.msg_name = &addr;
			/* if iov is set, it's allocated already */
			if (!kmsg->iov)
				kmsg->iov = kmsg->fast_iov;
			kmsg->msg.msg_iter.iov = kmsg->iov;
		} else {
			struct io_sr_msg *sr = &req->sr_msg;

			kmsg = &io.msg;
			kmsg->msg.msg_name = &addr;

			io.msg.iov = io.msg.fast_iov;
			ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
					sr->msg_flags, &io.msg.uaddr,
					&io.msg.iov);
			if (ret)
				return ret;
		}

		flags = req->sr_msg.msg_flags;
		if (flags & MSG_DONTWAIT)
			req->flags |= REQ_F_NOWAIT;
		else if (force_nonblock)
			flags |= MSG_DONTWAIT;

		ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
						kmsg->uaddr, flags);
		if (force_nonblock && ret == -EAGAIN) {
			if (req->io)
				return -EAGAIN;
			if (io_alloc_async_ctx(req)) {
				if (kmsg && kmsg->iov != kmsg->fast_iov)
					kfree(kmsg->iov);
				return -ENOMEM;
			}
			memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
			req->flags |= REQ_F_NEED_CLEANUP;
			return -EAGAIN;
		}
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
	}

	if (kmsg && kmsg->iov != kmsg->fast_iov)
		kfree(kmsg->iov);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	io_cqring_add_event(req, ret);
	if (ret < 0)
		req_set_fail_links(req);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
	return 0;
}


static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
		   bool force_nonblock)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw.kiocb;
	struct iov_iter iter;
	size_t iov_count;
	ssize_t io_size, ret;

	ret = io_import_iovec(READ, req, &iovec, &iter);
	if (ret < 0)
		return ret;

	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;

	req->result = 0;
	io_size = ret;
	if (req->flags & REQ_F_LINK)
		req->result = io_size;

	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
	if (force_nonblock && !io_file_supports_async(req->file)) {
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}

	iov_count = iov_iter_count(&iter);
	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
	if (!ret) {
		ssize_t ret2;

		if (req->file->f_op->read_iter)
			ret2 = call_read_iter(req->file, kiocb, &iter);
		else
			ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);

		/* Catch -EAGAIN return for forced non-blocking submission */
		if (!force_nonblock || ret2 != -EAGAIN) {
			kiocb_done(kiocb, ret2, nxt, req->in_async);
		} else {
copy_iov:
			ret = io_setup_async_rw(req, io_size, iovec,
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
	}
out_free:
	kfree(iovec);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	return ret;
}


p11_rpc_buffer_get_rsa_pkcs_oaep_mechanism_value (p11_buffer *buffer,
						  size_t *offset,
						  void *value,
						  CK_ULONG *value_length)
{
	uint64_t val[3];
	const unsigned char *data;
	size_t len;

	if (!p11_rpc_buffer_get_uint64 (buffer, offset, &val[0]))
		return false;
	if (!p11_rpc_buffer_get_uint64 (buffer, offset, &val[1]))
		return false;
	if (!p11_rpc_buffer_get_uint64 (buffer, offset, &val[2]))
		return false;
	if (!p11_rpc_buffer_get_byte_array (buffer, offset, &data, &len))
		return false;

	if (value) {
		CK_RSA_PKCS_OAEP_PARAMS params;

		params.hashAlg = val[0];
		params.mgf = val[1];
		params.source = val[2];
		params.pSourceData = (void *) data;
		params.ulSourceDataLen = len;

		memcpy (value, &params, sizeof (CK_RSA_PKCS_OAEP_PARAMS));
	}

	if (value_length)
		*value_length = sizeof (CK_RSA_PKCS_OAEP_PARAMS);

	return true;
}


p11_rpc_buffer_encode_uint16 (unsigned char* data,
                              uint16_t value)
{
	data[0] = (value >> 8) & 0xff;
	data[1] = (value >> 0) & 0xff;
}


p11_rpc_buffer_get_uint16 (p11_buffer *buf,
                           size_t *offset,
                           uint16_t *value)
{
	unsigned char *ptr;
	if (buf->len < 2 || *offset > buf->len - 2) {
		p11_buffer_fail (buf);
		return false;
	}
	ptr = (unsigned char*)buf->data + *offset;
	if (value != NULL)
		*value = p11_rpc_buffer_decode_uint16 (ptr);
	*offset = *offset + 2;
	return true;
}


p11_rpc_message_write_byte (p11_rpc_message *msg,
                            CK_BYTE val)
{
	assert (msg != NULL);
	assert (msg->output != NULL);

	/* Make sure this is in the right order */
	assert (!msg->signature || p11_rpc_message_verify_part (msg, "y"));
	p11_rpc_buffer_add_byte (msg->output, val);
	return !p11_buffer_failed (msg->output);
}


rpc_C_VerifyFinal (CK_X_FUNCTION_LIST *self,
                   p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR signature;
	CK_ULONG signature_len;

	BEGIN_CALL (VerifyFinal);
		IN_ULONG (session);
		IN_BYTE_ARRAY (signature, signature_len);
	PROCESS_CALL ((self, session, signature, signature_len));
	END_CALL;
}


proxy_C_UnwrapKey (CK_X_FUNCTION_LIST *self,
                   CK_SESSION_HANDLE handle,
                   CK_MECHANISM_PTR mechanism,
                   CK_OBJECT_HANDLE unwrapping_key,
                   CK_BYTE_PTR wrapped_key,
                   CK_ULONG wrapped_key_len,
                   CK_ATTRIBUTE_PTR template,
                   CK_ULONG count,
                   CK_OBJECT_HANDLE_PTR key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_UnwrapKey) (handle, mechanism, unwrapping_key, wrapped_key, wrapped_key_len, template, count, key);
}


rpc_C_Sign (CK_X_FUNCTION_LIST *self,
            p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR part;
	CK_ULONG part_len;
	CK_BYTE_PTR signature;
	CK_ULONG signature_len;

	BEGIN_CALL (Sign);
		IN_ULONG (session);
		IN_BYTE_ARRAY (part, part_len);
		IN_BYTE_BUFFER (signature, signature_len);
	PROCESS_CALL ((self, session, part, part_len, signature, &signature_len));
		OUT_BYTE_ARRAY (signature, signature_len);
	END_CALL;

}


rpc_C_GenerateKey (CK_X_FUNCTION_LIST *self,
                   p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_MECHANISM mechanism;
	CK_ATTRIBUTE_PTR template;
	CK_ULONG count;
	CK_OBJECT_HANDLE key;

	BEGIN_CALL (GenerateKey);
		IN_ULONG (session);
		IN_MECHANISM (mechanism);
		IN_ATTRIBUTE_ARRAY (template, count);
	PROCESS_CALL ((self, session, &mechanism, template, count, &key));
		OUT_ULONG (key);
	END_CALL;
}


proxy_C_VerifyInit (CK_X_FUNCTION_LIST *self,
                    CK_SESSION_HANDLE handle,
                    CK_MECHANISM_PTR mechanism,
                    CK_OBJECT_HANDLE key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_VerifyInit) (handle, mechanism, key);
}


alloc_size (int num)
{
	unsigned int n = num ? 1 : 0;
	while (n < num && n > 0)
		n <<= 1;
	return n;
}


p11_index_finish (p11_index *index)
{
	p11_dict *changes;
	index_object *obj;
	p11_dictiter iter;

	return_if_fail (index != NULL);

	if (!index->changes)
		return;

	changes = index->changes;
	index->changes = NULL;

	p11_dict_iterate (changes, &iter);
	while (p11_dict_next (&iter, NULL, (void **)&obj)) {
		index_notify (index, obj->handle, obj->attrs);
		obj->attrs = NULL;
	}

	p11_dict_free (changes);
}


sink_one_match (p11_index *index,
                index_object *obj,
                CK_ATTRIBUTE *match,
                CK_ULONG count,
                void *data)
{
	CK_OBJECT_HANDLE *result = data;

	if (p11_attrs_matchn (obj->attrs, match, count)) {
		*result = obj->handle;
		return false;
	}

	return true;
}


default_store (void *data,
               p11_index *index,
               CK_OBJECT_HANDLE handle,
               CK_ATTRIBUTE **attrs)
{
	return CKR_OK;
}


proxy_C_GetOperationState (CK_X_FUNCTION_LIST *self,
                           CK_SESSION_HANDLE handle,
                           CK_BYTE_PTR operation_state,
                           CK_ULONG_PTR operation_state_len)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_GetOperationState) (handle, operation_state, operation_state_len);
}


rpc_C_UnwrapKey (CK_X_FUNCTION_LIST *self,
                 p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_MECHANISM mechanism;
	CK_OBJECT_HANDLE unwrapping_key;
	CK_BYTE_PTR wrapped_key;
	CK_ULONG wrapped_key_len;
	CK_ATTRIBUTE_PTR template;
	CK_ULONG attribute_count;
	CK_OBJECT_HANDLE key;

	BEGIN_CALL (UnwrapKey);
		IN_ULONG (session);
		IN_MECHANISM (mechanism);
		IN_ULONG (unwrapping_key);
		IN_BYTE_ARRAY (wrapped_key, wrapped_key_len);
		IN_ATTRIBUTE_ARRAY (template, attribute_count);
	PROCESS_CALL ((self, session, &mechanism, unwrapping_key, wrapped_key, wrapped_key_len, template, attribute_count, &key));
		OUT_ULONG (key);
	END_CALL;
}


rpc_C_Digest (CK_X_FUNCTION_LIST *self,
              p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR data;
	CK_ULONG data_len;
	CK_BYTE_PTR digest;
	CK_ULONG digest_len;

	BEGIN_CALL (Digest);
		IN_ULONG (session);
		IN_BYTE_ARRAY (data, data_len);
		IN_BYTE_BUFFER (digest, digest_len);
	PROCESS_CALL ((self, session, data, data_len, digest, &digest_len));
		OUT_BYTE_ARRAY (digest, digest_len);
	END_CALL;
}


rpc_C_CopyObject (CK_X_FUNCTION_LIST *self,
                  p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_OBJECT_HANDLE object;
	CK_ATTRIBUTE_PTR template;
	CK_ULONG count;
	CK_OBJECT_HANDLE new_object;

	BEGIN_CALL (CopyObject);
		IN_ULONG (session);
		IN_ULONG (object);
		IN_ATTRIBUTE_ARRAY (template, count);
	PROCESS_CALL ((self, session, object, template, count, &new_object));
		OUT_ULONG (new_object);
	END_CALL;
}


rpc_C_DestroyObject (CK_X_FUNCTION_LIST *self,
                     p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_OBJECT_HANDLE object;

	BEGIN_CALL (DestroyObject);
		IN_ULONG (session);
		IN_ULONG (object);
	PROCESS_CALL ((self, session, object));
	END_CALL;
}


rpc_C_VerifyRecover (CK_X_FUNCTION_LIST *self,
                     p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR signature;
	CK_ULONG signature_len;
	CK_BYTE_PTR data;
	CK_ULONG data_len;

	BEGIN_CALL (VerifyRecover);
		IN_ULONG (session);
		IN_BYTE_ARRAY (signature, signature_len);
		IN_BYTE_BUFFER (data, data_len);
	PROCESS_CALL ((self, session, signature, signature_len, data, &data_len));
		OUT_BYTE_ARRAY (data, data_len);
	END_CALL;
}


proxy_C_WaitForSlotEvent (CK_X_FUNCTION_LIST *self,
                          CK_FLAGS flags,
                          CK_SLOT_ID_PTR slot,
                          CK_VOID_PTR reserved)
{
	State *state = (State *)self;
	Proxy *py = state->px;
	CK_FUNCTION_LIST_PTR *f;
	CK_FUNCTION_LIST_PTR funcs;
	CK_SLOT_ID real_slot;
	unsigned int i;
	CK_RV rv = CKR_NO_EVENT;

	/* Only the non-blocking case is supported. */
	if ((flags & CKF_DONT_BLOCK) == 0)
		return CKR_FUNCTION_NOT_SUPPORTED;

	p11_lock ();

	for (f = py->inited; *f; ++f) {
		funcs = *f;
		assert (funcs != NULL);

		rv = (funcs->C_WaitForSlotEvent) (flags, &real_slot, reserved);
		if (rv == CKR_NO_EVENT)
			continue;
		if (rv != CKR_OK)
			break;
		for (i = 0; i < py->n_mappings; i++)
			if (py->mappings[i].funcs == funcs &&
			    py->mappings[i].real_slot == real_slot) {
				*slot = py->mappings[i].wrap_slot;
				break;
			}
	}

	p11_unlock ();

	return rv;
}


rpc_C_GenerateKeyPair (CK_X_FUNCTION_LIST *self,
                       p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_MECHANISM mechanism;
	CK_ATTRIBUTE_PTR public_key_template;
	CK_ULONG public_key_attribute_count;
	CK_ATTRIBUTE_PTR private_key_template;
	CK_ULONG private_key_attribute_count;
	CK_OBJECT_HANDLE public_key;
	CK_OBJECT_HANDLE private_key;

	BEGIN_CALL (GenerateKeyPair);
		IN_ULONG (session);
		IN_MECHANISM (mechanism);
		IN_ATTRIBUTE_ARRAY (public_key_template, public_key_attribute_count);
		IN_ATTRIBUTE_ARRAY (private_key_template, private_key_attribute_count);
	PROCESS_CALL ((self, session, &mechanism, public_key_template, public_key_attribute_count, private_key_template, private_key_attribute_count, &public_key, &private_key));
		OUT_ULONG (public_key);
		OUT_ULONG (private_key);
	END_CALL;
}


getprogname (void)
{
	const char *name;

#if defined (HAVE_GETEXECNAME)
	const char *p;
	name = getexecname();
	p = strrchr (name ? name : "", '/');
	if (p != NULL)
		name = p + 1;
#elif defined (HAVE_PROGRAM_INVOCATION_SHORT_NAME)
#ifdef __linux__
	name = program_invocation_name;
	assert (name);
	if (*name == '/') {
		/*
		 * Some programs pack command line arguments into argv[0].
		 * Check if it is the case by reading /proc/self/exe and extract
		 * the program name.
		 *
		 * Logic borrowed from:
		 * <https://github.com/mesa3d/mesa/commit/759b94038987bb983398cd4b1d2cb1c8f79817a9>.
		 */
		static char *buf;

		if (!buf)
			buf = realpath ("/proc/self/exe", NULL);

		if (buf && strncmp (buf, name, strlen (buf)) == 0)
			/* Use the executable path if the prefix matches. */
			name = strrchr (buf, '/') + 1;
		else
			/* Otherwise fall back to
			 * program_invocation_short_name. */
			name = program_invocation_short_name;
	} else {
		name = program_invocation_short_name;
	}
#else
	name = program_invocation_short_name;
#endif
#elif defined (HAVE___PROGNAME)
	name = __progname;
#else
	#error No way to retrieve short program name
#endif

	return name;
}


evalsubshell(union node *n, int flags)
{
	struct job *jp;
	int backgnd = (n->type == NBACKGND);
	int status;

	errlinno = lineno = n->nredir.linno;
	if (funcline)
		lineno -= funcline - 1;

	expredir(n->nredir.redirect);
	INTOFF;
	if (!backgnd && flags & EV_EXIT && !have_traps()) {
		forkreset();
		goto nofork;
	}
	jp = makejob(n, 1);
	if (forkshell(jp, n, backgnd) == 0) {
		flags |= EV_EXIT;
		if (backgnd)
			flags &=~ EV_TESTED;
nofork:
		INTON;
		redirect(n->nredir.redirect, 0);
		evaltreenr(n->nredir.n, flags);
		/* never returns */
	}
	status = 0;
	if (! backgnd)
		status = waitforjob(jp);
	INTON;
	return status;
}


dotcmd(int argc, char **argv)
{
	int status = 0;

	nextopt(nullstr);
	argv = argptr;

	if (*argv) {
		char *fullname;

		fullname = find_dot_file(*argv);
		setinputfile(fullname, INPUT_PUSH_FILE);
		commandname = fullname;
		status = cmdloop(0);
		popfile();
	}

	return status;
}


static inline void rb_time_split(u64 val, unsigned long *top, unsigned long *bottom)
{
	*top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
	*bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
}


static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
		      struct ring_buffer_event *event)
{
	local_inc(&cpu_buffer->entries);
	rb_end_commit(cpu_buffer);
}


u64 ring_buffer_time_stamp(struct trace_buffer *buffer, int cpu)
{
	u64 time;

	preempt_disable_notrace();
	time = rb_time_stamp(buffer);
	preempt_enable_notrace();

	return time;
}


unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long ret;

	if (!cpumask_test_cpu(cpu, buffer->cpumask))
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;

	return ret;
}


rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
		  struct ring_buffer_event *event)
{
	unsigned long new_index, old_index;
	struct buffer_page *bpage;
	unsigned long index;
	unsigned long addr;
	u64 write_stamp;
	u64 delta;

	new_index = rb_event_index(event);
	old_index = new_index + rb_event_ts_length(event);
	addr = (unsigned long)event;
	addr &= PAGE_MASK;

	bpage = READ_ONCE(cpu_buffer->tail_page);

	delta = rb_time_delta(event);

	if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp))
		return 0;

	/* Make sure the write stamp is read before testing the location */
	barrier();

	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
		unsigned long write_mask =
			local_read(&bpage->write) & ~RB_WRITE_MASK;
		unsigned long event_length = rb_event_length(event);

		/* Something came in, can't discard */
		if (!rb_time_cmpxchg(&cpu_buffer->write_stamp,
				       write_stamp, write_stamp - delta))
			return 0;

		/*
		 * If an event were to come in now, it would see that the
		 * write_stamp and the before_stamp are different, and assume
		 * that this event just added itself before updating
		 * the write stamp. The interrupting event will fix the
		 * write stamp for us, and use the before stamp as its delta.
		 */

		/*
		 * This is on the tail page. It is possible that
		 * a write could come in and move the tail page
		 * and write to the next page. That is fine
		 * because we just shorten what is on this page.
		 */
		old_index += write_mask;
		new_index += write_mask;
		index = local_cmpxchg(&bpage->write, old_index, new_index);
		if (index == old_index) {
			/* update counters */
			local_sub(event_length, &cpu_buffer->entries_bytes);
			return 1;
		}
	}

	/* could not discard */
	return 0;
}


rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
{
	rb_head_page_deactivate(cpu_buffer);

	cpu_buffer->head_page
		= list_entry(cpu_buffer->pages, struct buffer_page, list);
	local_set(&cpu_buffer->head_page->write, 0);
	local_set(&cpu_buffer->head_page->entries, 0);
	local_set(&cpu_buffer->head_page->page->commit, 0);

	cpu_buffer->head_page->read = 0;

	cpu_buffer->tail_page = cpu_buffer->head_page;
	cpu_buffer->commit_page = cpu_buffer->head_page;

	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
	INIT_LIST_HEAD(&cpu_buffer->new_pages);
	local_set(&cpu_buffer->reader_page->write, 0);
	local_set(&cpu_buffer->reader_page->entries, 0);
	local_set(&cpu_buffer->reader_page->page->commit, 0);
	cpu_buffer->reader_page->read = 0;

	local_set(&cpu_buffer->entries_bytes, 0);
	local_set(&cpu_buffer->overrun, 0);
	local_set(&cpu_buffer->commit_overrun, 0);
	local_set(&cpu_buffer->dropped_events, 0);
	local_set(&cpu_buffer->entries, 0);
	local_set(&cpu_buffer->committing, 0);
	local_set(&cpu_buffer->commits, 0);
	local_set(&cpu_buffer->pages_touched, 0);
	local_set(&cpu_buffer->pages_read, 0);
	cpu_buffer->last_pages_touch = 0;
	cpu_buffer->shortest_full = 0;
	cpu_buffer->read = 0;
	cpu_buffer->read_bytes = 0;

	rb_time_set(&cpu_buffer->write_stamp, 0);
	rb_time_set(&cpu_buffer->before_stamp, 0);

	cpu_buffer->lost_events = 0;
	cpu_buffer->last_overrun = 0;

	rb_head_page_activate(cpu_buffer);
}


rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
	     unsigned long tail, struct rb_event_info *info)
{
	struct buffer_page *tail_page = info->tail_page;
	struct buffer_page *commit_page = cpu_buffer->commit_page;
	struct trace_buffer *buffer = cpu_buffer->buffer;
	struct buffer_page *next_page;
	int ret;

	next_page = tail_page;

	rb_inc_page(cpu_buffer, &next_page);

	/*
	 * If for some reason, we had an interrupt storm that made
	 * it all the way around the buffer, bail, and warn
	 * about it.
	 */
	if (unlikely(next_page == commit_page)) {
		local_inc(&cpu_buffer->commit_overrun);
		goto out_reset;
	}

	/*
	 * This is where the fun begins!
	 *
	 * We are fighting against races between a reader that
	 * could be on another CPU trying to swap its reader
	 * page with the buffer head.
	 *
	 * We are also fighting against interrupts coming in and
	 * moving the head or tail on us as well.
	 *
	 * If the next page is the head page then we have filled
	 * the buffer, unless the commit page is still on the
	 * reader page.
	 */
	if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) {

		/*
		 * If the commit is not on the reader page, then
		 * move the header page.
		 */
		if (!rb_is_reader_page(cpu_buffer->commit_page)) {
			/*
			 * If we are not in overwrite mode,
			 * this is easy, just stop here.
			 */
			if (!(buffer->flags & RB_FL_OVERWRITE)) {
				local_inc(&cpu_buffer->dropped_events);
				goto out_reset;
			}

			ret = rb_handle_head_page(cpu_buffer,
						  tail_page,
						  next_page);
			if (ret < 0)
				goto out_reset;
			if (ret)
				goto out_again;
		} else {
			/*
			 * We need to be careful here too. The
			 * commit page could still be on the reader
			 * page. We could have a small buffer, and
			 * have filled up the buffer with events
			 * from interrupts and such, and wrapped.
			 *
			 * Note, if the tail page is also the on the
			 * reader_page, we let it move out.
			 */
			if (unlikely((cpu_buffer->commit_page !=
				      cpu_buffer->tail_page) &&
				     (cpu_buffer->commit_page ==
				      cpu_buffer->reader_page))) {
				local_inc(&cpu_buffer->commit_overrun);
				goto out_reset;
			}
		}
	}

	rb_tail_page_update(cpu_buffer, tail_page, next_page);

 out_again:

	rb_reset_tail(cpu_buffer, tail, info);

	/* Commit what we have for now. */
	rb_end_commit(cpu_buffer);
	/* rb_end_commit() decs committing */
	local_inc(&cpu_buffer->committing);

	/* fail and let the caller try again */
	return ERR_PTR(-EAGAIN);

 out_reset:
	/* reset write */
	rb_reset_tail(cpu_buffer, tail, info);

	return NULL;
}


rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
{
	struct trace_buffer *buffer;
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
	int nr_loops = 0;

	if (ts)
		*ts = 0;

	cpu_buffer = iter->cpu_buffer;
	buffer = cpu_buffer->buffer;

	/*
	 * Check if someone performed a consuming read to
	 * the buffer. A consuming read invalidates the iterator
	 * and we need to reset the iterator in this case.
	 */
	if (unlikely(iter->cache_read != cpu_buffer->read ||
		     iter->cache_reader_page != cpu_buffer->reader_page))
		rb_iter_reset(iter);

 again:
	if (ring_buffer_iter_empty(iter))
		return NULL;

	/*
	 * As the writer can mess with what the iterator is trying
	 * to read, just give up if we fail to get an event after
	 * three tries. The iterator is not as reliable when reading
	 * the ring buffer with an active write as the consumer is.
	 * Do not warn if the three failures is reached.
	 */
	if (++nr_loops > 3)
		return NULL;

	if (rb_per_cpu_empty(cpu_buffer))
		return NULL;

	if (iter->head >= rb_page_size(iter->head_page)) {
		rb_inc_iter(iter);
		goto again;
	}

	event = rb_iter_head_event(iter);
	if (!event)
		goto again;

	switch (event->type_len) {
	case RINGBUF_TYPE_PADDING:
		if (rb_null_event(event)) {
			rb_inc_iter(iter);
			goto again;
		}
		rb_advance_iter(iter);
		return event;

	case RINGBUF_TYPE_TIME_EXTEND:
		/* Internal data, OK to advance */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_TIME_STAMP:
		if (ts) {
			*ts = ring_buffer_event_time_stamp(event);
			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
							 cpu_buffer->cpu, ts);
		}
		/* Internal data, OK to advance */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_DATA:
		if (ts && !(*ts)) {
			*ts = iter->read_stamp + event->time_delta;
			ring_buffer_normalize_time_stamp(buffer,
							 cpu_buffer->cpu, ts);
		}
		return event;

	default:
		RB_WARN_ON(cpu_buffer, 1);
	}

	return NULL;
}


static int io_files_update(struct io_kiocb *req, bool force_nonblock,
			   struct io_comp_state *cs)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_uring_files_update up;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	up.offset = req->files_update.offset;
	up.fds = req->files_update.arg;

	mutex_lock(&ctx->uring_lock);
	ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
	mutex_unlock(&ctx->uring_lock);

	if (ret < 0)
		req_set_fail_links(req);
	__io_req_complete(req, ret, 0, cs);
	return 0;
}


static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
{
	/* pure poll stashes this in ->io, poll driven retry elsewhere */
	if (req->opcode == IORING_OP_POLL_ADD)
		return (struct io_poll_iocb *) req->io;
	return req->apoll->double_poll;
}


static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
{
	int i, j;

	if (!ctx->user_bufs)
		return -ENXIO;

	for (i = 0; i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *imu = &ctx->user_bufs[i];

		for (j = 0; j < imu->nr_bvecs; j++)
			unpin_user_page(imu->bvec[j].bv_page);

		io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
		kvfree(imu->bvec);
		imu->nr_bvecs = 0;
	}

	kfree(ctx->user_bufs);
	ctx->user_bufs = NULL;
	ctx->nr_user_bufs = 0;
	return 0;
}


	__acquires(ctx->uring_lock)
{
	int ret;

	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

	if (io_register_op_must_quiesce(opcode)) {
		percpu_ref_kill(&ctx->refs);

		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
		ret = wait_for_completion_interruptible(&ctx->ref_comp);
		mutex_lock(&ctx->uring_lock);
		if (ret) {
			percpu_ref_resurrect(&ctx->refs);
			ret = -EINTR;
			goto out;
		}
	}

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffer_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffer_unregister(ctx);
		break;
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_sqe_files_update(ctx, arg, nr_args);
		break;
	case IORING_REGISTER_EVENTFD:
	case IORING_REGISTER_EVENTFD_ASYNC:
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	if (io_register_op_must_quiesce(opcode)) {
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
out:
		reinit_completion(&ctx->ref_comp);
	}
	return ret;
}


static bool io_timeout_remove_link(struct io_ring_ctx *ctx,
				   struct io_kiocb *req)
{
	struct io_kiocb *preq;
	bool found = false;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry(preq, &ctx->timeout_list, timeout.list) {
		found = io_match_link(preq, req);
		if (found) {
			__io_timeout_cancel(preq);
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	return found;
}


static void __io_state_file_put(struct io_submit_state *state)
{
	if (state->has_refs)
		fput_many(state->file, state->has_refs);
	state->file = NULL;
}


static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			   bool is_timeout_link)
{
	struct io_timeout_data *data;
	unsigned flags;
	u32 off = READ_ONCE(sqe->off);

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
		return -EINVAL;
	if (off && is_timeout_link)
		return -EINVAL;
	flags = READ_ONCE(sqe->timeout_flags);
	if (flags & ~IORING_TIMEOUT_ABS)
		return -EINVAL;

	req->timeout.off = off;

	if (!req->io && io_alloc_async_ctx(req))
		return -ENOMEM;

	data = &req->io->timeout;
	data->req = req;

	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
		return -EFAULT;

	if (flags & IORING_TIMEOUT_ABS)
		data->mode = HRTIMER_MODE_ABS;
	else
		data->mode = HRTIMER_MODE_REL;

	hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
	return 0;
}


static struct file *pick_file(struct files_struct *files, unsigned fd)
{
	struct file *file = NULL;
	struct fdtable *fdt;

	spin_lock(&files->file_lock);
	fdt = files_fdtable(files);
	if (fd >= fdt->max_fds)
		goto out_unlock;
	file = fdt->fd[fd];
	if (!file)
		goto out_unlock;
	rcu_assign_pointer(fdt->fd[fd], NULL);
	__put_unused_fd(files, fd);

out_unlock:
	spin_unlock(&files->file_lock);
	return file;
}


static void io_req_task_file_table_put(struct callback_head *cb)
{
	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	struct fs_struct *fs = req->work.fs;

	spin_lock(&req->work.fs->lock);
	if (--fs->users)
		fs = NULL;
	spin_unlock(&req->work.fs->lock);
	if (fs)
		free_fs_struct(fs);
	req->work.fs = NULL;
	__io_free_req_finish(req);
}


static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
{
	if (req->opcode == IORING_OP_POLL_ADD)
		return &req->poll;
	return &req->apoll->poll;
}


static inline void io_uring_free(struct task_struct *tsk)
{
}


static int do_execve(struct filename *filename,
	const char __user *const __user *__argv,
	const char __user *const __user *__envp)
{
	struct user_arg_ptr argv = { .ptr.native = __argv };
	struct user_arg_ptr envp = { .ptr.native = __envp };
	return do_execveat_common(AT_FDCWD, filename, argv, envp, 0);
}


static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
			   __poll_t mask, task_work_func_t func)
{
	bool twa_signal_ok;
	int ret;

	/* for instances that support it check for an event match first: */
	if (mask && !(mask & poll->events))
		return 0;

	trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);

	list_del_init(&poll->wait.entry);

	req->result = mask;
	init_task_work(&req->task_work, func);
	percpu_ref_get(&req->ctx->refs);

	/*
	 * If we using the signalfd wait_queue_head for this wakeup, then
	 * it's not safe to use TWA_SIGNAL as we could be recursing on the
	 * tsk->sighand->siglock on doing the wakeup. Should not be needed
	 * either, as the normal wakeup will suffice.
	 */
	twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh);

	/*
	 * If this fails, then the task is exiting. When a task exits, the
	 * work gets canceled, so just cancel this request as well instead
	 * of executing it. We can't safely execute it anyway, as we may not
	 * have the needed state needed for it anyway.
	 */
	ret = io_req_task_work_add(req, &req->task_work, twa_signal_ok);
	if (unlikely(ret)) {
		struct task_struct *tsk;

		WRITE_ONCE(poll->canceled, true);
		tsk = io_wq_get_task(req->ctx->io_wq);
		task_work_add(tsk, &req->task_work, 0);
		wake_up_process(tsk);
	}
	return 1;
}


static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_poll_remove_double(req);
	req->poll.done = true;
	io_cqring_fill_event(req, error ? error : mangle_poll(mask));
	io_commit_cqring(ctx);
}


static void __io_free_req(struct io_kiocb *req)
{
	if (!io_dismantle_req(req)) {
		__io_free_req_finish(req);
	} else {
		int ret;

		init_task_work(&req->task_work, io_req_task_file_table_put);
		ret = task_work_add(req->task, &req->task_work, TWA_RESUME);
		if (unlikely(ret)) {
			struct task_struct *tsk;

			tsk = io_wq_get_task(req->ctx->io_wq);
			task_work_add(tsk, &req->task_work, 0);
		}
	}
}


static inline void io_consume_sqe(struct io_ring_ctx *ctx)
{
	ctx->cached_sq_head++;
}


SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const sigset_t __user *, sig,
		size_t, sigsz)
{
	struct io_ring_ctx *ctx;
	long ret = -EBADF;
	int submitted = 0;
	struct fd f;

	io_run_task_work();

	if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
		return -EINVAL;

	f = fdget(fd);
	if (!f.file)
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (f.file->f_op != &io_uring_fops)
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (!percpu_ref_tryget(&ctx->refs))
		goto out_fput;

	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
	ret = 0;
	if (ctx->flags & IORING_SETUP_SQPOLL) {
		if (!list_empty_careful(&ctx->cq_overflow_list))
			io_cqring_overflow_flush(ctx, false, NULL, NULL);
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sqo_wait);
		submitted = to_submit;
	} else if (to_submit) {
		ret = io_uring_add_task_file(f.file);
		if (unlikely(ret))
			goto out;
		mutex_lock(&ctx->uring_lock);
		submitted = io_submit_sqes(ctx, to_submit);
		mutex_unlock(&ctx->uring_lock);

		if (submitted != to_submit)
			goto out;
	}
	if (flags & IORING_ENTER_GETEVENTS) {
		min_complete = min(min_complete, ctx->cq_entries);

		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
			ret = io_iopoll_check(ctx, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
		}
	}

out:
	percpu_ref_put(&ctx->refs);
out_fput:
	fdput(f);
	return submitted ? submitted : ret;
}


static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
			   struct iov_iter *iter)
{
	ssize_t ret = 0;

	/*
	 * Don't support polled IO through this interface, and we can't
	 * support non-blocking either. For the latter, this just causes
	 * the kiocb to be handled from an async context.
	 */
	if (kiocb->ki_flags & IOCB_HIPRI)
		return -EOPNOTSUPP;
	if (kiocb->ki_flags & IOCB_NOWAIT)
		return -EAGAIN;

	while (iov_iter_count(iter)) {
		struct iovec iovec;
		ssize_t nr;

		if (!iov_iter_is_bvec(iter)) {
			iovec = iov_iter_iovec(iter);
		} else {
			/* fixed buffers import bvec */
			iovec.iov_base = kmap(iter->bvec->bv_page)
						+ iter->iov_offset;
			iovec.iov_len = min(iter->count,
					iter->bvec->bv_len - iter->iov_offset);
		}

		if (rw == READ) {
			nr = file->f_op->read(file, iovec.iov_base,
					      iovec.iov_len, io_kiocb_ppos(kiocb));
		} else {
			nr = file->f_op->write(file, iovec.iov_base,
					       iovec.iov_len, io_kiocb_ppos(kiocb));
		}

		if (iov_iter_is_bvec(iter))
			kunmap(iter->bvec->bv_page);

		if (nr < 0) {
			if (!ret)
				ret = nr;
			break;
		}
		ret += nr;
		if (nr != iovec.iov_len)
			break;
		iov_iter_advance(iter, nr);
	}

	return ret;
}


static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
			 struct io_comp_state *cs)
{
	int ret;

	ret = io_req_defer(req, sqe);
	if (ret) {
		if (ret != -EIOCBQUEUED) {
fail_req:
			req_set_fail_links(req);
			io_put_req(req);
			io_req_complete(req, ret);
		}
	} else if (req->flags & REQ_F_FORCE_ASYNC) {
		if (!req->io) {
			ret = io_req_defer_prep(req, sqe);
			if (unlikely(ret))
				goto fail_req;
		}

		/*
		 * Never try inline submit of IOSQE_ASYNC is set, go straight
		 * to async execution.
		 */
		io_req_init_async(req);
		req->work.flags |= IO_WQ_WORK_CONCURRENT;
		io_queue_async_work(req);
	} else {
		__io_queue_sqe(req, sqe, cs);
	}
}


static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{
	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);

	complete(&ctx->ref_comp);
}


static bool io_req_clean_work(struct io_kiocb *req)
{
	if (!(req->flags & REQ_F_WORK_INITIALIZED))
		return false;

	req->flags &= ~REQ_F_WORK_INITIALIZED;

	if (req->work.mm) {
		mmdrop(req->work.mm);
		req->work.mm = NULL;
	}
	if (req->work.creds) {
		put_cred(req->work.creds);
		req->work.creds = NULL;
	}
	if (req->work.fs) {
		struct fs_struct *fs = req->work.fs;

		if (req->flags & REQ_F_COMP_LOCKED)
			return true;

		spin_lock(&req->work.fs->lock);
		if (--fs->users)
			fs = NULL;
		spin_unlock(&req->work.fs->lock);
		if (fs)
			free_fs_struct(fs);
		req->work.fs = NULL;
	}

	return false;
}


static int io_connect(struct io_kiocb *req, bool force_nonblock,
		      struct io_comp_state *cs)
{
	struct io_async_ctx __io, *io;
	unsigned file_flags;
	int ret;

	if (req->io) {
		io = req->io;
	} else {
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.connect.address);
		if (ret)
			goto out;
		io = &__io;
	}

	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->connect.address,
					req->connect.addr_len, file_flags);
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
		if (req->io)
			return -EAGAIN;
		if (io_alloc_async_ctx(req)) {
			ret = -ENOMEM;
			goto out;
		}
		memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
		return -EAGAIN;
	}
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
out:
	if (ret < 0)
		req_set_fail_links(req);
	__io_req_complete(req, ret, 0, cs);
	return 0;
}


static bool io_uring_cancel_files(struct io_ring_ctx *ctx,
				  struct files_struct *files)
{
	if (list_empty_careful(&ctx->inflight_list))
		return false;

	io_cancel_defer_files(ctx, files);
	/* cancel all at once, should be faster than doing it one by one*/
	io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);

	while (!list_empty_careful(&ctx->inflight_list)) {
		struct io_kiocb *cancel_req = NULL, *req;
		DEFINE_WAIT(wait);

		spin_lock_irq(&ctx->inflight_lock);
		list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
			if (files && req->work.files != files)
				continue;
			/* req is being completed, ignore */
			if (!refcount_inc_not_zero(&req->refs))
				continue;
			cancel_req = req;
			break;
		}
		if (cancel_req)
			prepare_to_wait(&ctx->inflight_wait, &wait,
						TASK_UNINTERRUPTIBLE);
		spin_unlock_irq(&ctx->inflight_lock);

		/* We need to keep going until we don't find a matching req */
		if (!cancel_req)
			break;
		/* cancel this request, or head link requests */
		io_attempt_cancel(ctx, cancel_req);
		io_put_req(cancel_req);
		/* cancellations _may_ trigger task work */
		io_run_task_work();
		schedule();
		finish_wait(&ctx->inflight_wait, &wait);
	}

	return true;
}


static void zynq_slcr_reset_hold(Object *obj)
{
    ZynqSLCRState *s = ZYNQ_SLCR(obj);

    /* will disable all output clocks */
    zynq_slcr_compute_clocks(s);
    zynq_slcr_propagate_clocks(s);
}


static uint64_t zynq_slcr_read(void *opaque, hwaddr offset,
    unsigned size)
{
    ZynqSLCRState *s = opaque;
    offset /= 4;
    uint32_t ret = s->regs[offset];

    if (!zynq_slcr_check_offset(offset, true)) {
        qemu_log_mask(LOG_GUEST_ERROR, "zynq_slcr: Invalid read access to "
                      " addr %" HWADDR_PRIx "\n", offset * 4);
    }

    DB_PRINT("addr: %08" HWADDR_PRIx " data: %08" PRIx32 "\n", offset * 4, ret);
    return ret;
}


static void zynq_slcr_ps_clk_callback(void *opaque)
{
    ZynqSLCRState *s = (ZynqSLCRState *) opaque;
    zynq_slcr_compute_clocks(s);
    zynq_slcr_propagate_clocks(s);
}


static long tty_compat_ioctl(struct file *file, unsigned int cmd,
				unsigned long arg)
{
	struct tty_struct *tty = file_tty(file);
	struct tty_ldisc *ld;
	int retval = -ENOIOCTLCMD;

	switch (cmd) {
	case TIOCOUTQ:
	case TIOCSTI:
	case TIOCGWINSZ:
	case TIOCSWINSZ:
	case TIOCGEXCL:
	case TIOCGETD:
	case TIOCSETD:
	case TIOCGDEV:
	case TIOCMGET:
	case TIOCMSET:
	case TIOCMBIC:
	case TIOCMBIS:
	case TIOCGICOUNT:
	case TIOCGPGRP:
	case TIOCSPGRP:
	case TIOCGSID:
	case TIOCSERGETLSR:
	case TIOCGRS485:
	case TIOCSRS485:
#ifdef TIOCGETP
	case TIOCGETP:
	case TIOCSETP:
	case TIOCSETN:
#endif
#ifdef TIOCGETC
	case TIOCGETC:
	case TIOCSETC:
#endif
#ifdef TIOCGLTC
	case TIOCGLTC:
	case TIOCSLTC:
#endif
	case TCSETSF:
	case TCSETSW:
	case TCSETS:
	case TCGETS:
#ifdef TCGETS2
	case TCGETS2:
	case TCSETSF2:
	case TCSETSW2:
	case TCSETS2:
#endif
	case TCGETA:
	case TCSETAF:
	case TCSETAW:
	case TCSETA:
	case TIOCGLCKTRMIOS:
	case TIOCSLCKTRMIOS:
#ifdef TCGETX
	case TCGETX:
	case TCSETX:
	case TCSETXW:
	case TCSETXF:
#endif
	case TIOCGSOFTCAR:
	case TIOCSSOFTCAR:

	case PPPIOCGCHAN:
	case PPPIOCGUNIT:
		return tty_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
	case TIOCCONS:
	case TIOCEXCL:
	case TIOCNXCL:
	case TIOCVHANGUP:
	case TIOCSBRK:
	case TIOCCBRK:
	case TCSBRK:
	case TCSBRKP:
	case TCFLSH:
	case TIOCGPTPEER:
	case TIOCNOTTY:
	case TIOCSCTTY:
	case TCXONC:
	case TIOCMIWAIT:
	case TIOCSERCONFIG:
		return tty_ioctl(file, cmd, arg);
	}

	if (tty_paranoia_check(tty, file_inode(file), "tty_ioctl"))
		return -EINVAL;

	switch (cmd) {
	case TIOCSSERIAL:
		return compat_tty_tiocsserial(tty, compat_ptr(arg));
	case TIOCGSERIAL:
		return compat_tty_tiocgserial(tty, compat_ptr(arg));
	}
	if (tty->ops->compat_ioctl) {
		retval = tty->ops->compat_ioctl(tty, cmd, arg);
		if (retval != -ENOIOCTLCMD)
			return retval;
	}

	ld = tty_ldisc_ref_wait(tty);
	if (!ld)
		return hung_up_tty_compat_ioctl(file, cmd, arg);
	if (ld->ops->compat_ioctl)
		retval = ld->ops->compat_ioctl(tty, file, cmd, arg);
	if (retval == -ENOIOCTLCMD && ld->ops->ioctl)
		retval = ld->ops->ioctl(tty, file,
				(unsigned long)compat_ptr(cmd), arg);
	tty_ldisc_deref(ld);

	return retval;
}


void tty_release_struct(struct tty_struct *tty, int idx)
{
	/*
	 * Ask the line discipline code to release its structures
	 */
	tty_ldisc_release(tty);

	/* Wait for pending work before tty destruction commmences */
	tty_flush_works(tty);

	tty_debug_hangup(tty, "freeing structure\n");
	/*
	 * The release_tty function takes care of the details of clearing
	 * the slots and preserving the termios structure.
	 */
	mutex_lock(&tty_mutex);
	release_tty(tty, idx);
	mutex_unlock(&tty_mutex);
}


int tty_hung_up_p(struct file *filp)
{
	return (filp && filp->f_op == &hung_up_tty_fops);
}


static __poll_t tty_poll(struct file *filp, poll_table *wait)
{
	struct tty_struct *tty = file_tty(filp);
	struct tty_ldisc *ld;
	__poll_t ret = 0;

	if (tty_paranoia_check(tty, file_inode(filp), "tty_poll"))
		return 0;

	ld = tty_ldisc_ref_wait(tty);
	if (!ld)
		return hung_up_tty_poll(filp, wait);
	if (ld->ops->poll)
		ret = ld->ops->poll(tty, filp, wait);
	tty_ldisc_deref(ld);
	return ret;
}


on_reauthentication_conversation_started (GdmSession              *session,
                                          const char              *service_name,
                                          ReauthenticationRequest *request)
{
        g_debug ("GdmSessionWorker: reauthentication service '%s' started",
                 service_name);
}


gdm_session_worker_authorize_user (GdmSessionWorker *worker,
                                   gboolean          password_is_required,
                                   GError          **error)
{
        int error_code;
        int authentication_flags;

        g_debug ("GdmSessionWorker: determining if authenticated user (password required:%d) is authorized to session",
                 password_is_required);

        authentication_flags = 0;

        if (password_is_required) {
                authentication_flags |= PAM_DISALLOW_NULL_AUTHTOK;
        }

        /* check that the account isn't disabled or expired
         */
        error_code = pam_acct_mgmt (worker->priv->pam_handle, authentication_flags);

        /* it's possible that the user needs to change their password or pin code
         */
        if (error_code == PAM_NEW_AUTHTOK_REQD && !worker->priv->is_program_session) {
                g_debug ("GdmSessionWorker: authenticated user requires new auth token");
                error_code = pam_chauthtok (worker->priv->pam_handle, PAM_CHANGE_EXPIRED_AUTHTOK);

                gdm_session_worker_get_username (worker, NULL);

                if (error_code != PAM_SUCCESS) {
                        gdm_session_auditor_report_password_change_failure (worker->priv->auditor);
                } else {
                        gdm_session_auditor_report_password_changed (worker->priv->auditor);
                }
        }

        /* If the user is reauthenticating, then authorization isn't required to
         * proceed, the user is already logged in after all.
         */
        if (worker->priv->is_reauth_session) {
                error_code = PAM_SUCCESS;
        }

        if (error_code != PAM_SUCCESS) {
                g_debug ("GdmSessionWorker: user is not authorized to log in: %s",
                         pam_strerror (worker->priv->pam_handle, error_code));
                g_set_error (error,
                             GDM_SESSION_WORKER_ERROR,
                             GDM_SESSION_WORKER_ERROR_AUTHORIZING,
                             "%s", get_friendly_error_message (error_code));
                goto out;
        }

        g_debug ("GdmSessionWorker: state AUTHORIZED");
        gdm_session_worker_set_state (worker, GDM_SESSION_WORKER_STATE_AUTHORIZED);

 out:
        if (error_code != PAM_SUCCESS) {
                gdm_session_worker_uninitialize_pam (worker, error_code);
                return FALSE;
        }

        return TRUE;
}


wait_until_dbus_signal_emission_to_manager_finishes (GdmSessionWorker *worker)
{
        g_autoptr (GdmDBusPeer) peer_proxy = NULL;
        g_autoptr (GError) error = NULL;
        gboolean pinged;

        peer_proxy = gdm_dbus_peer_proxy_new_sync (worker->priv->connection,
                                                   G_DBUS_PROXY_FLAGS_DO_NOT_LOAD_PROPERTIES,
                                                   NULL,
                                                   "/org/freedesktop/DBus",
                                                   NULL,
                                                   &error);

        if (peer_proxy == NULL) {
                g_debug ("GdmSessionWorker: could not create peer proxy to daemon: %s",
                         error->message);
                return;
        }

        pinged = gdm_dbus_peer_call_ping_sync (peer_proxy, NULL, &error);

        if (!pinged) {
                g_debug ("GdmSessionWorker: could not ping daemon: %s",
                         error->message);
                return;
        }
}


gdm_session_worker_accredit_user (GdmSessionWorker  *worker,
                                  GError           **error)
{
        gboolean ret;
        gboolean res;
        uid_t    uid;
        gid_t    gid;
        char    *shell;
        char    *home;
        int      error_code;

        ret = FALSE;

        home = NULL;
        shell = NULL;

        if (worker->priv->username == NULL) {
                g_debug ("GdmSessionWorker: Username not set");
                error_code = PAM_USER_UNKNOWN;
                g_set_error (error,
                             GDM_SESSION_WORKER_ERROR,
                             GDM_SESSION_WORKER_ERROR_GIVING_CREDENTIALS,
                             _("no user account available"));
                goto out;
        }

        uid = 0;
        gid = 0;
        res = _lookup_passwd_info (worker->priv->username,
                                   &uid,
                                   &gid,
                                   &home,
                                   &shell);
        if (! res) {
                g_debug ("GdmSessionWorker: Unable to lookup account info");
                error_code = PAM_AUTHINFO_UNAVAIL;
                g_set_error (error,
                             GDM_SESSION_WORKER_ERROR,
                             GDM_SESSION_WORKER_ERROR_GIVING_CREDENTIALS,
                             _("no user account available"));
                goto out;
        }

        gdm_session_worker_update_environment_from_passwd_info (worker,
                                                                uid,
                                                                gid,
                                                                home,
                                                                shell);

        /* Let's give the user a default PATH if he doesn't already have one
         */
        if (!gdm_session_worker_environment_variable_is_set (worker, "PATH")) {
                if (strcmp (BINDIR, "/usr/bin") == 0) {
                        gdm_session_worker_set_environment_variable (worker, "PATH",
                                                                     GDM_SESSION_DEFAULT_PATH);
                } else {
                        gdm_session_worker_set_environment_variable (worker, "PATH",
                                                                     BINDIR ":" GDM_SESSION_DEFAULT_PATH);
                }
        }

        if (! _change_user (worker, uid, gid)) {
                g_debug ("GdmSessionWorker: Unable to change to user");
                error_code = PAM_SYSTEM_ERR;
                g_set_error (error, GDM_SESSION_WORKER_ERROR,
                             GDM_SESSION_WORKER_ERROR_GIVING_CREDENTIALS,
                             "%s", _("Unable to change to user"));
                goto out;
        }

        error_code = pam_setcred (worker->priv->pam_handle, worker->priv->cred_flags);

        /* If the user is reauthenticating and they've made it this far, then there
         * is no reason we should lock them out of their session.  They've already
         * proved they are they same person who logged in, and that's all we care
         * about.
         */
        if (worker->priv->is_reauth_session) {
                error_code = PAM_SUCCESS;
        }

        if (error_code != PAM_SUCCESS) {
                g_set_error (error,
                             GDM_SESSION_WORKER_ERROR,
                             GDM_SESSION_WORKER_ERROR_GIVING_CREDENTIALS,
                             "%s",
                             pam_strerror (worker->priv->pam_handle, error_code));
                goto out;
        }

        ret = TRUE;

 out:
        g_free (home);
        g_free (shell);
        if (ret) {
                g_debug ("GdmSessionWorker: state ACCREDITED");
                ret = TRUE;

                gdm_session_worker_get_username (worker, NULL);
                gdm_session_auditor_report_user_accredited (worker->priv->auditor);
                gdm_session_worker_set_state (worker, GDM_SESSION_WORKER_STATE_ACCREDITED);
        } else {
                gdm_session_worker_uninitialize_pam (worker, error_code);
        }

        return ret;
}


gdm_session_worker_pam_new_messages_handler (int                        number_of_messages,
                                             const struct pam_message **messages,
                                             struct pam_response      **responses,
                                             GdmSessionWorker          *worker)
{
        struct pam_response *replies;
        int                  return_value;
        int                  i;

        g_debug ("GdmSessionWorker: %d new messages received from PAM\n", number_of_messages);

        return_value = PAM_CONV_ERR;

        if (number_of_messages < 0) {
                return PAM_CONV_ERR;
        }

        if (number_of_messages == 0) {
                if (responses) {
                        *responses = NULL;
                }

                return PAM_SUCCESS;
        }

        /* we want to generate one reply for every question
         */
        replies = (struct pam_response *) calloc (number_of_messages,
                                                  sizeof (struct pam_response));
        for (i = 0; i < number_of_messages; i++) {
                gboolean got_response;
                char    *response;

                response = NULL;
                got_response = gdm_session_worker_process_pam_message (worker,
                                                                       messages[i],
                                                                       &response);
                if (!got_response) {
                        goto out;
                }

                replies[i].resp = response;
                replies[i].resp_retcode = PAM_SUCCESS;
        }

        return_value = PAM_SUCCESS;

 out:
        if (return_value != PAM_SUCCESS) {
                for (i = 0; i < number_of_messages; i++) {
                        if (replies[i].resp != NULL) {
                                memset (replies[i].resp, 0, strlen (replies[i].resp));
                                free (replies[i].resp);
                        }
                        memset (&replies[i], 0, sizeof (replies[i]));
                }
                free (replies);
                replies = NULL;
        }

        if (responses) {
                *responses = replies;
        }

        g_debug ("GdmSessionWorker: PAM conversation returning %d: %s",
                 return_value,
                 pam_strerror (worker->priv->pam_handle, return_value));

        return return_value;
}


gdm_session_worker_set_environment_variable (GdmSessionWorker *worker,
                                             const char       *key,
                                             const char       *value)
{
        int error_code;
        char *environment_entry;

        if (value != NULL) {
                environment_entry = g_strdup_printf ("%s=%s", key, value);
        } else {
                /* empty value means "remove from environment" */
                environment_entry = g_strdup (key);
        }

        error_code = pam_putenv (worker->priv->pam_handle,
                                 environment_entry);

        if (error_code != PAM_SUCCESS) {
                g_warning ("cannot put %s in pam environment: %s\n",
                           environment_entry,
                           pam_strerror (worker->priv->pam_handle, error_code));
        }
        g_debug ("GdmSessionWorker: Set PAM environment variable: '%s'", environment_entry);
        g_free (environment_entry);
}


gdm_session_worker_handle_establish_credentials (GdmDBusWorker         *object,
                                                 GDBusMethodInvocation *invocation)
{
        GdmSessionWorker *worker = GDM_SESSION_WORKER (object);
        validate_and_queue_state_change (worker, invocation, GDM_SESSION_WORKER_STATE_ACCREDITED);

        if (!worker->priv->is_reauth_session) {
                worker->priv->cred_flags = PAM_ESTABLISH_CRED;
        } else {
                worker->priv->cred_flags = PAM_REINITIALIZE_CRED;
        }

        return TRUE;
}


gdm_session_worker_finalize (GObject *object)
{
        GdmSessionWorker *worker;

        g_return_if_fail (object != NULL);
        g_return_if_fail (GDM_IS_SESSION_WORKER (object));

        worker = GDM_SESSION_WORKER (object);

        g_return_if_fail (worker->priv != NULL);

        gdm_session_worker_unwatch_child (worker);

        if (worker->priv->child_pid > 0) {
                gdm_signal_pid (worker->priv->child_pid, SIGTERM);
                gdm_wait_on_pid (worker->priv->child_pid);
        }

        if (worker->priv->pam_handle != NULL) {
                gdm_session_worker_uninitialize_pam (worker, PAM_SUCCESS);
        }

        jump_back_to_initial_vt (worker);

        g_object_unref (worker->priv->user_settings);
        g_free (worker->priv->service);
        g_free (worker->priv->x11_display_name);
        g_free (worker->priv->x11_authority_file);
        g_free (worker->priv->display_device);
        g_free (worker->priv->display_seat_id);
        g_free (worker->priv->hostname);
        g_free (worker->priv->username);
        g_free (worker->priv->server_address);
        g_strfreev (worker->priv->arguments);
        g_strfreev (worker->priv->extensions);

        g_hash_table_unref (worker->priv->reauthentication_requests);

        G_OBJECT_CLASS (gdm_session_worker_parent_class)->finalize (object);
}


gdm_session_worker_load_env_d (GdmSessionWorker *worker)
{
        GFile *dir;

        dir = g_file_new_for_path (DATADIR "/gdm/env.d");
        gdm_session_worker_load_env_dir (worker, dir);
        g_object_unref (dir);

        dir = g_file_new_for_path (GDMCONFDIR "/env.d");
        gdm_session_worker_load_env_dir (worker, dir);
        g_object_unref (dir);
}


do_start_session (GdmSessionWorker *worker)
{
        GError  *error;
        gboolean res;

        error = NULL;
        res = gdm_session_worker_start_session (worker, &error);
        if (res) {
                gdm_dbus_worker_complete_start_program (GDM_DBUS_WORKER (worker),
                                                        worker->priv->pending_invocation,
                                                        worker->priv->child_pid);
        } else {
                g_dbus_method_invocation_take_error (worker->priv->pending_invocation, error);
        }
        worker->priv->pending_invocation = NULL;
}


reauthentication_request_new (GdmSessionWorker      *worker,
                              GPid                   pid_of_caller,
                              uid_t                  uid_of_caller,
                              GDBusMethodInvocation *invocation)
{
        ReauthenticationRequest *request;
        const char * const * environment;
        const char *address;

        environment = gdm_session_worker_get_environment (worker);

        request = g_slice_new (ReauthenticationRequest);

        request->worker = worker;
        request->pid_of_caller = pid_of_caller;
        request->uid_of_caller = uid_of_caller;
        request->session = gdm_session_new (GDM_SESSION_VERIFICATION_MODE_REAUTHENTICATE,
                                            uid_of_caller,
                                            worker->priv->x11_display_name,
                                            worker->priv->hostname,
                                            worker->priv->display_device,
                                            worker->priv->display_seat_id,
                                            worker->priv->x11_authority_file,
                                            worker->priv->display_is_local,
                                            environment);

        g_signal_connect (request->session,
                          "client-connected",
                          G_CALLBACK (on_reauthentication_client_connected),
                          request);
        g_signal_connect (request->session,
                          "client-disconnected",
                          G_CALLBACK (on_reauthentication_client_disconnected),
                          request);
        g_signal_connect (request->session,
                          "cancelled",
                          G_CALLBACK (on_reauthentication_cancelled),
                          request);
        g_signal_connect (request->session,
                          "conversation-started",
                          G_CALLBACK (on_reauthentication_conversation_started),
                          request);
        g_signal_connect (request->session,
                          "conversation-stopped",
                          G_CALLBACK (on_reauthentication_conversation_stopped),
                          request);
        g_signal_connect (request->session,
                          "verification-complete",
                          G_CALLBACK (on_reauthentication_verification_complete),
                          request);

        address = gdm_session_get_server_address (request->session);

        gdm_dbus_worker_complete_start_reauthentication (GDM_DBUS_WORKER (worker),
                                                         invocation,
                                                         address);

        return request;
}


gdm_session_worker_get_environment (GdmSessionWorker *worker)
{
        return (const char * const *) pam_getenvlist (worker->priv->pam_handle);
}


static OPJ_BOOL opj_j2k_read_poc(opj_j2k_t *p_j2k,
                                 OPJ_BYTE * p_header_data,
                                 OPJ_UINT32 p_header_size,
                                 opj_event_mgr_t * p_manager
                                )
{
    OPJ_UINT32 i, l_nb_comp, l_tmp;
    opj_image_t * l_image = 00;
    OPJ_UINT32 l_old_poc_nb, l_current_poc_nb, l_current_poc_remaining;
    OPJ_UINT32 l_chunk_size, l_comp_room;

    opj_cp_t *l_cp = 00;
    opj_tcp_t *l_tcp = 00;
    opj_poc_t *l_current_poc = 00;

    /* preconditions */
    assert(p_header_data != 00);
    assert(p_j2k != 00);
    assert(p_manager != 00);

    l_image = p_j2k->m_private_image;
    l_nb_comp = l_image->numcomps;
    if (l_nb_comp <= 256) {
        l_comp_room = 1;
    } else {
        l_comp_room = 2;
    }
    l_chunk_size = 5 + 2 * l_comp_room;
    l_current_poc_nb = p_header_size / l_chunk_size;
    l_current_poc_remaining = p_header_size % l_chunk_size;

    if ((l_current_poc_nb <= 0) || (l_current_poc_remaining != 0)) {
        opj_event_msg(p_manager, EVT_ERROR, "Error reading POC marker\n");
        return OPJ_FALSE;
    }

    l_cp = &(p_j2k->m_cp);
    l_tcp = (p_j2k->m_specific_param.m_decoder.m_state == J2K_STATE_TPH) ?
            &l_cp->tcps[p_j2k->m_current_tile_number] :
            p_j2k->m_specific_param.m_decoder.m_default_tcp;
    l_old_poc_nb = l_tcp->POC ? l_tcp->numpocs + 1 : 0;
    l_current_poc_nb += l_old_poc_nb;

    if (l_current_poc_nb >= J2K_MAX_POCS) {
        opj_event_msg(p_manager, EVT_ERROR, "Too many POCs %d\n", l_current_poc_nb);
        return OPJ_FALSE;
    }

    /* now poc is in use.*/
    l_tcp->POC = 1;

    l_current_poc = &l_tcp->pocs[l_old_poc_nb];
    for (i = l_old_poc_nb; i < l_current_poc_nb; ++i) {
        opj_read_bytes(p_header_data, &(l_current_poc->resno0),
                       1);                               /* RSpoc_i */
        ++p_header_data;
        opj_read_bytes(p_header_data, &(l_current_poc->compno0),
                       l_comp_room);  /* CSpoc_i */
        p_header_data += l_comp_room;
        opj_read_bytes(p_header_data, &(l_current_poc->layno1),
                       2);                               /* LYEpoc_i */
        /* make sure layer end is in acceptable bounds */
        l_current_poc->layno1 = opj_uint_min(l_current_poc->layno1, l_tcp->numlayers);
        p_header_data += 2;
        opj_read_bytes(p_header_data, &(l_current_poc->resno1),
                       1);                               /* REpoc_i */
        ++p_header_data;
        opj_read_bytes(p_header_data, &(l_current_poc->compno1),
                       l_comp_room);  /* CEpoc_i */
        p_header_data += l_comp_room;
        opj_read_bytes(p_header_data, &l_tmp,
                       1);                                                                 /* Ppoc_i */
        ++p_header_data;
        l_current_poc->prg = (OPJ_PROG_ORDER) l_tmp;
        /* make sure comp is in acceptable bounds */
        l_current_poc->compno1 = opj_uint_min(l_current_poc->compno1, l_nb_comp);
        ++l_current_poc;
    }

    l_tcp->numpocs = l_current_poc_nb - 1;
    return OPJ_TRUE;
}


void opj_j2k_destroy(opj_j2k_t *p_j2k)
{
    if (p_j2k == 00) {
        return;
    }

    if (p_j2k->m_is_decoder) {

        if (p_j2k->m_specific_param.m_decoder.m_default_tcp != 00) {
            opj_j2k_tcp_destroy(p_j2k->m_specific_param.m_decoder.m_default_tcp);
            opj_free(p_j2k->m_specific_param.m_decoder.m_default_tcp);
            p_j2k->m_specific_param.m_decoder.m_default_tcp = 00;
        }

        if (p_j2k->m_specific_param.m_decoder.m_header_data != 00) {
            opj_free(p_j2k->m_specific_param.m_decoder.m_header_data);
            p_j2k->m_specific_param.m_decoder.m_header_data = 00;
            p_j2k->m_specific_param.m_decoder.m_header_data_size = 0;
        }

        opj_free(p_j2k->m_specific_param.m_decoder.m_comps_indices_to_decode);
        p_j2k->m_specific_param.m_decoder.m_comps_indices_to_decode = 00;
        p_j2k->m_specific_param.m_decoder.m_numcomps_to_decode = 0;

    } else {

        if (p_j2k->m_specific_param.m_encoder.m_encoded_tile_data) {
            opj_free(p_j2k->m_specific_param.m_encoder.m_encoded_tile_data);
            p_j2k->m_specific_param.m_encoder.m_encoded_tile_data = 00;
        }

        if (p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer) {
            opj_free(p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer);
            p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer = 00;
            p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_current = 00;
        }

        if (p_j2k->m_specific_param.m_encoder.m_header_tile_data) {
            opj_free(p_j2k->m_specific_param.m_encoder.m_header_tile_data);
            p_j2k->m_specific_param.m_encoder.m_header_tile_data = 00;
            p_j2k->m_specific_param.m_encoder.m_header_tile_data_size = 0;
        }
    }

    opj_tcd_destroy(p_j2k->m_tcd);

    opj_j2k_cp_destroy(&(p_j2k->m_cp));
    memset(&(p_j2k->m_cp), 0, sizeof(opj_cp_t));

    opj_procedure_list_destroy(p_j2k->m_procedure_list);
    p_j2k->m_procedure_list = 00;

    opj_procedure_list_destroy(p_j2k->m_validation_list);
    p_j2k->m_procedure_list = 00;

    j2k_destroy_cstr_index(p_j2k->cstr_index);
    p_j2k->cstr_index = NULL;

    opj_image_destroy(p_j2k->m_private_image);
    p_j2k->m_private_image = NULL;

    opj_image_destroy(p_j2k->m_output_image);
    p_j2k->m_output_image = NULL;

    opj_thread_pool_destroy(p_j2k->m_tp);
    p_j2k->m_tp = NULL;

    opj_free(p_j2k);
}


static OPJ_BOOL opj_j2k_move_data_from_codec_to_output_image(opj_j2k_t * p_j2k,
        opj_image_t * p_image)
{
    OPJ_UINT32 compno;

    /* Move data and copy one information from codec to output image*/
    if (p_j2k->m_specific_param.m_decoder.m_numcomps_to_decode > 0) {
        opj_image_comp_t* newcomps =
            (opj_image_comp_t*) opj_malloc(
                p_j2k->m_specific_param.m_decoder.m_numcomps_to_decode *
                sizeof(opj_image_comp_t));
        if (newcomps == NULL) {
            opj_image_destroy(p_j2k->m_private_image);
            p_j2k->m_private_image = NULL;
            return OPJ_FALSE;
        }
        for (compno = 0; compno < p_image->numcomps; compno++) {
            opj_image_data_free(p_image->comps[compno].data);
            p_image->comps[compno].data = NULL;
        }
        for (compno = 0;
                compno < p_j2k->m_specific_param.m_decoder.m_numcomps_to_decode; compno++) {
            OPJ_UINT32 src_compno =
                p_j2k->m_specific_param.m_decoder.m_comps_indices_to_decode[compno];
            memcpy(&(newcomps[compno]),
                   &(p_j2k->m_output_image->comps[src_compno]),
                   sizeof(opj_image_comp_t));
            newcomps[compno].resno_decoded =
                p_j2k->m_output_image->comps[src_compno].resno_decoded;
            newcomps[compno].data = p_j2k->m_output_image->comps[src_compno].data;
            p_j2k->m_output_image->comps[src_compno].data = NULL;
        }
        for (compno = 0; compno < p_image->numcomps; compno++) {
            assert(p_j2k->m_output_image->comps[compno].data == NULL);
            opj_image_data_free(p_j2k->m_output_image->comps[compno].data);
            p_j2k->m_output_image->comps[compno].data = NULL;
        }
        p_image->numcomps = p_j2k->m_specific_param.m_decoder.m_numcomps_to_decode;
        opj_free(p_image->comps);
        p_image->comps = newcomps;
    } else {
        for (compno = 0; compno < p_image->numcomps; compno++) {
            p_image->comps[compno].resno_decoded =
                p_j2k->m_output_image->comps[compno].resno_decoded;
            opj_image_data_free(p_image->comps[compno].data);
            p_image->comps[compno].data = p_j2k->m_output_image->comps[compno].data;
#if 0
            char fn[256];
            sprintf(fn, "/tmp/%d.raw", compno);
            FILE *debug = fopen(fn, "wb");
            fwrite(p_image->comps[compno].data, sizeof(OPJ_INT32),
                   p_image->comps[compno].w * p_image->comps[compno].h, debug);
            fclose(debug);
#endif
            p_j2k->m_output_image->comps[compno].data = NULL;
        }
    }
    return OPJ_TRUE;
}


static OPJ_BOOL opj_jp2_setup_encoding_validation(opj_jp2_t *jp2,
        opj_event_mgr_t * p_manager)
{
    /* preconditions */
    assert(jp2 != 00);
    assert(p_manager != 00);

    if (! opj_procedure_list_add_procedure(jp2->m_validation_list,
                                           (opj_procedure)opj_jp2_default_validation, p_manager)) {
        return OPJ_FALSE;
    }
    /* DEVELOPER CORNER, add your custom validation procedure */

    return OPJ_TRUE;
}


static OPJ_BOOL opj_j2k_update_rates(opj_j2k_t *p_j2k,
                                     opj_stream_private_t *p_stream,
                                     opj_event_mgr_t * p_manager)
{
    opj_cp_t * l_cp = 00;
    opj_image_t * l_image = 00;
    opj_tcp_t * l_tcp = 00;
    opj_image_comp_t * l_img_comp = 00;

    OPJ_UINT32 i, j, k;
    OPJ_INT32 l_x0, l_y0, l_x1, l_y1;
    OPJ_FLOAT32 * l_rates = 0;
    OPJ_FLOAT32 l_sot_remove;
    OPJ_UINT32 l_bits_empty, l_size_pixel;
    OPJ_UINT32 l_tile_size = 0;
    OPJ_UINT32 l_last_res;
    OPJ_FLOAT32(* l_tp_stride_func)(opj_tcp_t *) = 00;

    /* preconditions */
    assert(p_j2k != 00);
    assert(p_manager != 00);
    assert(p_stream != 00);

    OPJ_UNUSED(p_manager);

    l_cp = &(p_j2k->m_cp);
    l_image = p_j2k->m_private_image;
    l_tcp = l_cp->tcps;

    l_bits_empty = 8 * l_image->comps->dx * l_image->comps->dy;
    l_size_pixel = l_image->numcomps * l_image->comps->prec;
    l_sot_remove = (OPJ_FLOAT32) opj_stream_tell(p_stream) / (OPJ_FLOAT32)(
                       l_cp->th * l_cp->tw);

    if (l_cp->m_specific_param.m_enc.m_tp_on) {
        l_tp_stride_func = opj_j2k_get_tp_stride;
    } else {
        l_tp_stride_func = opj_j2k_get_default_stride;
    }

    for (i = 0; i < l_cp->th; ++i) {
        for (j = 0; j < l_cp->tw; ++j) {
            OPJ_FLOAT32 l_offset = (OPJ_FLOAT32)(*l_tp_stride_func)(l_tcp) /
                                   (OPJ_FLOAT32)l_tcp->numlayers;

            /* 4 borders of the tile rescale on the image if necessary */
            l_x0 = opj_int_max((OPJ_INT32)(l_cp->tx0 + j * l_cp->tdx),
                               (OPJ_INT32)l_image->x0);
            l_y0 = opj_int_max((OPJ_INT32)(l_cp->ty0 + i * l_cp->tdy),
                               (OPJ_INT32)l_image->y0);
            l_x1 = opj_int_min((OPJ_INT32)(l_cp->tx0 + (j + 1) * l_cp->tdx),
                               (OPJ_INT32)l_image->x1);
            l_y1 = opj_int_min((OPJ_INT32)(l_cp->ty0 + (i + 1) * l_cp->tdy),
                               (OPJ_INT32)l_image->y1);

            l_rates = l_tcp->rates;

            /* Modification of the RATE >> */
            if (*l_rates > 0.0f) {
                *l_rates = (((OPJ_FLOAT32)(l_size_pixel * (OPJ_UINT32)(l_x1 - l_x0) *
                                           (OPJ_UINT32)(l_y1 - l_y0)))
                            /
                            ((*l_rates) * (OPJ_FLOAT32)l_bits_empty)
                           )
                           -
                           l_offset;
            }

            ++l_rates;

            for (k = 1; k < l_tcp->numlayers; ++k) {
                if (*l_rates > 0.0f) {
                    *l_rates = (((OPJ_FLOAT32)(l_size_pixel * (OPJ_UINT32)(l_x1 - l_x0) *
                                               (OPJ_UINT32)(l_y1 - l_y0)))
                                /
                                ((*l_rates) * (OPJ_FLOAT32)l_bits_empty)
                               )
                               -
                               l_offset;
                }

                ++l_rates;
            }

            ++l_tcp;

        }
    }

    l_tcp = l_cp->tcps;

    for (i = 0; i < l_cp->th; ++i) {
        for (j = 0; j < l_cp->tw; ++j) {
            l_rates = l_tcp->rates;

            if (*l_rates > 0.0f) {
                *l_rates -= l_sot_remove;

                if (*l_rates < 30.0f) {
                    *l_rates = 30.0f;
                }
            }

            ++l_rates;

            l_last_res = l_tcp->numlayers - 1;

            for (k = 1; k < l_last_res; ++k) {

                if (*l_rates > 0.0f) {
                    *l_rates -= l_sot_remove;

                    if (*l_rates < * (l_rates - 1) + 10.0f) {
                        *l_rates  = (*(l_rates - 1)) + 20.0f;
                    }
                }

                ++l_rates;
            }

            if (*l_rates > 0.0f) {
                *l_rates -= (l_sot_remove + 2.f);

                if (*l_rates < * (l_rates - 1) + 10.0f) {
                    *l_rates  = (*(l_rates - 1)) + 20.0f;
                }
            }

            ++l_tcp;
        }
    }

    l_img_comp = l_image->comps;
    l_tile_size = 0;

    for (i = 0; i < l_image->numcomps; ++i) {
        l_tile_size += (opj_uint_ceildiv(l_cp->tdx, l_img_comp->dx)
                        *
                        opj_uint_ceildiv(l_cp->tdy, l_img_comp->dy)
                        *
                        l_img_comp->prec
                       );

        ++l_img_comp;
    }

    /* TODO: where does this magic value come from ? */
    /* This used to be 1.3 / 8, but with random data and very small code */
    /* block sizes, this is not enough. For example with */
    /* bin/test_tile_encoder 1 256 256 32 32 8 0 reversible_with_precinct.j2k 4 4 3 0 0 1 16 16 */
    /* TODO revise this to take into account the overhead linked to the */
    /* number of packets and number of code blocks in packets */
    l_tile_size = (OPJ_UINT32)(l_tile_size * 1.4 / 8);

    /* Arbitrary amount to make the following work: */
    /* bin/test_tile_encoder 1 256 256 17 16 8 0 reversible_no_precinct.j2k 4 4 3 0 0 1 */
    l_tile_size += 500;

    l_tile_size += opj_j2k_get_specific_header_sizes(p_j2k);

    p_j2k->m_specific_param.m_encoder.m_encoded_tile_size = l_tile_size;
    p_j2k->m_specific_param.m_encoder.m_encoded_tile_data =
        (OPJ_BYTE *) opj_malloc(p_j2k->m_specific_param.m_encoder.m_encoded_tile_size);
    if (p_j2k->m_specific_param.m_encoder.m_encoded_tile_data == 00) {
        return OPJ_FALSE;
    }

    if (OPJ_IS_CINEMA(l_cp->rsiz) || OPJ_IS_IMF(l_cp->rsiz)) {
        p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer =
            (OPJ_BYTE *) opj_malloc(5 *
                                    p_j2k->m_specific_param.m_encoder.m_total_tile_parts);
        if (! p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer) {
            return OPJ_FALSE;
        }

        p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_current =
            p_j2k->m_specific_param.m_encoder.m_tlm_sot_offsets_buffer;
    }

    return OPJ_TRUE;
}


void OPJ_CALLCONV opj_destroy_cstr_index(opj_codestream_index_t **p_cstr_index)
{
    if (*p_cstr_index) {
        j2k_destroy_cstr_index(*p_cstr_index);
        (*p_cstr_index) = NULL;
    }
}