int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
{
	int error;
	struct kobject *kobj = &dev->kobj;

	dev = get_device(dev);
	if (!dev)
		return -EINVAL;

	/*
	 * Change the kobject and the default attributes and groups of the
	 * ktype associated with it to @kuid/@kgid.
	 */
	error = sysfs_change_owner(kobj, kuid, kgid);
	if (error)
		goto out;

	/*
	 * Change the uevent file for @dev to the new owner. The uevent file
	 * was created in a separate step when @dev got added and we mirror
	 * that step here.
	 */
	error = sysfs_file_change_owner(kobj, dev_attr_uevent.attr.name, kuid,
					kgid);
	if (error)
		goto out;

	/*
	 * Change the device groups, the device groups associated with the
	 * device class, and the groups associated with the device type of @dev
	 * to @kuid/@kgid.
	 */
	error = device_attrs_change_owner(dev, kuid, kgid);
	if (error)
		goto out;

	error = dpm_sysfs_change_owner(dev, kuid, kgid);
	if (error)
		goto out;

#ifdef CONFIG_BLOCK
	if (sysfs_deprecated && dev->class == &block_class)
		goto out;
#endif

	/*
	 * Change the owner of the symlink located in the class directory of
	 * the device class associated with @dev which points to the actual
	 * directory entry for @dev to @kuid/@kgid. This ensures that the
	 * symlink shows the same permissions as its target.
	 */
	error = sysfs_link_change_owner(&dev->class->p->subsys.kobj, &dev->kobj,
					dev_name(dev), kuid, kgid);
	if (error)
		goto out;

out:
	put_device(dev);
	return error;
}


static ssize_t timeout_show(struct class *class, struct class_attribute *attr,
			    char *buf)
{
	return sprintf(buf, "%d\n", __firmware_loading_timeout());
}


static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj,
				   struct bin_attribute *bin_attr,
				   char *buffer, loff_t offset, size_t count)
{
	struct device *dev = kobj_to_dev(kobj);
	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
	struct fw_priv *fw_priv;
	ssize_t retval;

	if (!capable(CAP_SYS_RAWIO))
		return -EPERM;

	mutex_lock(&fw_lock);
	fw_priv = fw_sysfs->fw_priv;
	if (!fw_priv || fw_sysfs_done(fw_priv)) {
		retval = -ENODEV;
		goto out;
	}

	if (fw_priv->data) {
		if (offset + count > fw_priv->allocated_size) {
			retval = -ENOMEM;
			goto out;
		}
		firmware_rw_data(fw_priv, buffer, offset, count, false);
		retval = count;
	} else {
		retval = fw_realloc_pages(fw_sysfs, offset + count);
		if (retval)
			goto out;

		retval = count;
		firmware_rw(fw_priv, buffer, offset, count, false);
	}

	fw_priv->size = max_t(size_t, offset + count, fw_priv->size);
out:
	mutex_unlock(&fw_lock);
	return retval;
}


static ssize_t allocation_policy_show(struct device *dev,
				      struct device_attribute *attr, char *buf)
{
	struct cacheinfo *this_leaf = dev_get_drvdata(dev);
	unsigned int ci_attr = this_leaf->attributes;
	int n = 0;

	if ((ci_attr & CACHE_READ_ALLOCATE) && (ci_attr & CACHE_WRITE_ALLOCATE))
		n = sysfs_emit(buf, "ReadWriteAllocate\n");
	else if (ci_attr & CACHE_READ_ALLOCATE)
		n = sysfs_emit(buf, "ReadAllocate\n");
	else if (ci_attr & CACHE_WRITE_ALLOCATE)
		n = sysfs_emit(buf, "WriteAllocate\n");
	return n;
}


struct device *soc_device_to_device(struct soc_device *soc_dev)
{
	return &soc_dev->dev;
}


static void driver_deferred_probe_add_trigger(struct device *dev,
					      int local_trigger_count)
{
	driver_deferred_probe_add(dev);
	/* Did a trigger occur while probing? Need to re-trigger if yes */
	if (local_trigger_count != atomic_read(&deferred_trigger_count))
		driver_deferred_probe_trigger();
}


const struct soc_device_attribute *soc_device_match(
	const struct soc_device_attribute *matches)
{
	int ret = 0;

	if (!matches)
		return NULL;

	while (!ret) {
		if (!(matches->machine || matches->family ||
		      matches->revision || matches->soc_id))
			break;
		ret = bus_for_each_dev(&soc_bus_type, NULL, (void *)matches,
				       soc_device_match_one);
		if (ret < 0 && early_soc_dev_attr)
			ret = soc_device_match_attr(early_soc_dev_attr,
						    matches);
		if (ret < 0)
			return NULL;
		if (!ret)
			matches++;
		else
			return matches;
	}
	return NULL;
}


static struct fw_sysfs *to_fw_sysfs(struct device *dev)
{
	return container_of(dev, struct fw_sysfs, dev);
}


int register_cpu(struct cpu *cpu, int num)
{
	int error;

	cpu->node_id = cpu_to_node(num);
	memset(&cpu->dev, 0x00, sizeof(struct device));
	cpu->dev.id = num;
	cpu->dev.bus = &cpu_subsys;
	cpu->dev.release = cpu_device_release;
	cpu->dev.offline_disabled = !cpu->hotpluggable;
	cpu->dev.offline = !cpu_online(num);
	cpu->dev.of_node = of_get_cpu_node(num, NULL);
#ifdef CONFIG_GENERIC_CPU_AUTOPROBE
	cpu->dev.bus->uevent = cpu_uevent;
#endif
	cpu->dev.groups = common_cpu_attr_groups;
	if (cpu->hotpluggable)
		cpu->dev.groups = hotplugable_cpu_attr_groups;
	error = device_register(&cpu->dev);
	if (error) {
		put_device(&cpu->dev);
		return error;
	}

	per_cpu(cpu_sys_devices, num) = &cpu->dev;
	register_cpu_under_node(num, cpu_to_node(num));
	dev_pm_qos_expose_latency_limit(&cpu->dev,
					PM_QOS_RESUME_LATENCY_NO_CONSTRAINT);

	return 0;
}


struct device *cpu_device_create(struct device *parent, void *drvdata,
				 const struct attribute_group **groups,
				 const char *fmt, ...)
{
	va_list vargs;
	struct device *dev;

	va_start(vargs, fmt);
	dev = __cpu_device_create(parent, drvdata, groups, fmt, vargs);
	va_end(vargs);
	return dev;
}


static inline int cache_setup_of_node(unsigned int cpu) { return 0; }


static ssize_t wakeup_max_time_ms_show(struct device *dev,
				       struct device_attribute *attr, char *buf)
{
	s64 msec = 0;
	bool enabled = false;

	spin_lock_irq(&dev->power.lock);
	if (dev->power.wakeup) {
		msec = ktime_to_ms(dev->power.wakeup->max_time);
		enabled = true;
	}
	spin_unlock_irq(&dev->power.lock);
	return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n");
}


int pm_qos_sysfs_add_flags(struct device *dev)
{
	return sysfs_merge_group(&dev->kobj, &pm_qos_flags_attr_group);
}


int pm_qos_sysfs_add_resume_latency(struct device *dev)
{
	return sysfs_merge_group(&dev->kobj, &pm_qos_resume_latency_attr_group);
}


static ssize_t sync_state_only_show(struct device *dev,
				    struct device_attribute *attr, char *buf)
{
	struct device_link *link = to_devlink(dev);

	return sysfs_emit(buf, "%d\n",
			  !!(link->flags & DL_FLAG_SYNC_STATE_ONLY));
}


static void firmware_rw(struct fw_priv *fw_priv, char *buffer,
			loff_t offset, size_t count, bool read)
{
	while (count) {
		void *page_data;
		int page_nr = offset >> PAGE_SHIFT;
		int page_ofs = offset & (PAGE_SIZE-1);
		int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count);

		page_data = kmap(fw_priv->pages[page_nr]);

		if (read)
			memcpy(buffer, page_data + page_ofs, page_cnt);
		else
			memcpy(page_data + page_ofs, buffer, page_cnt);

		kunmap(fw_priv->pages[page_nr]);
		buffer += page_cnt;
		offset += page_cnt;
		count -= page_cnt;
	}
}


static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
			      char *buf)
{
	return sysfs_emit(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE));
}


static ssize_t online_store(struct device *dev, struct device_attribute *attr,
			    const char *buf, size_t count)
{
	bool val;
	int ret;

	ret = strtobool(buf, &val);
	if (ret < 0)
		return ret;

	ret = lock_device_hotplug_sysfs();
	if (ret)
		return ret;

	ret = val ? device_online(dev) : device_offline(dev);
	unlock_device_hotplug();
	return ret < 0 ? ret : count;
}


static int __init soc_bus_register(void)
{
	int ret;

	ret = bus_register(&soc_bus_type);
	if (ret)
		return ret;

	if (early_soc_dev_attr)
		return PTR_ERR(soc_device_register(early_soc_dev_attr));

	return 0;
}


static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
					   struct cacheinfo *sib_leaf)
{
	/*
	 * For non-DT/ACPI systems, assume unique level 1 caches, system-wide
	 * shared caches for all other levels. This will be used only if
	 * arch specific code has not populated shared_cpu_map
	 */
	return !(this_leaf->level == 1);
}


static x3f_huffnode_t *new_node(x3f_hufftree_t *tree)
{
	if (tree->free_node_index >= tree->total_node_index)
		throw LIBRAW_EXCEPTION_IO_CORRUPT;
  x3f_huffnode_t *t = &tree->nodes[tree->free_node_index];

  t->branch[0] = NULL;
  t->branch[1] = NULL;
  t->leaf = UNDEFINED_LEAF;

  tree->free_node_index++;

  return t;
}


void LibRaw::process_Sony_0x940e(uchar *buf, ushort len, unsigned long long id)
{
  if (((imSony.CameraType != LIBRAW_SONY_SLT) &&
       (imSony.CameraType != LIBRAW_SONY_ILCA)) ||
      (id == SonyID_SLT_A33)  ||
      (id == SonyID_SLT_A55) ||
      (id == SonyID_SLT_A35)  ||
      (len < 3))
    return;

  imSony.AFType = SonySubstitution[buf[0x02]];

  if (imSony.CameraType == LIBRAW_SONY_ILCA)
  {
    if (len >= 0x06)
    {
      imgdata.shootinginfo.FocusMode = SonySubstitution[buf[0x05]];
    }
    if (len >= 0x0051)
    {
      imSony.AFMicroAdjValue = SonySubstitution[buf[0x0050]];
    }
  }
  else
  {
    if (len >= 0x0c)
    {
      imgdata.shootinginfo.FocusMode = SonySubstitution[buf[0x0b]];
    }
    if (len >= 0x017e)
    {
      imSony.AFMicroAdjValue = SonySubstitution[buf[0x017d]];
    }
  }

  if (imSony.AFMicroAdjValue != 0)
    imSony.AFMicroAdjOn = 1;
}


merged_2v_upsample(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
                   JDIMENSION *in_row_group_ctr,
                   JDIMENSION in_row_groups_avail, JSAMPARRAY output_buf,
                   JDIMENSION *out_row_ctr, JDIMENSION out_rows_avail)
/* 2:1 vertical sampling case: may need a spare row. */
{
  my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample;
  JSAMPROW work_ptrs[2];
  JDIMENSION num_rows;          /* number of rows returned to caller */

  if (upsample->spare_full) {
    /* If we have a spare row saved from a previous cycle, just return it. */
    JDIMENSION size = upsample->out_row_width;
    if (cinfo->out_color_space == JCS_RGB565)
      size = cinfo->output_width * 2;
    jcopy_sample_rows(&upsample->spare_row, 0, output_buf + *out_row_ctr, 0, 1,
                      size);
    num_rows = 1;
    upsample->spare_full = FALSE;
  } else {
    /* Figure number of rows to return to caller. */
    num_rows = 2;
    /* Not more than the distance to the end of the image. */
    if (num_rows > upsample->rows_to_go)
      num_rows = upsample->rows_to_go;
    /* And not more than what the client can accept: */
    out_rows_avail -= *out_row_ctr;
    if (num_rows > out_rows_avail)
      num_rows = out_rows_avail;
    /* Create output pointer array for upsampler. */
    work_ptrs[0] = output_buf[*out_row_ctr];
    if (num_rows > 1) {
      work_ptrs[1] = output_buf[*out_row_ctr + 1];
    } else {
      work_ptrs[1] = upsample->spare_row;
      upsample->spare_full = TRUE;
    }
    /* Now do the upsampling. */
    (*upsample->upmethod) (cinfo, input_buf, *in_row_group_ctr, work_ptrs);
  }

  /* Adjust counts */
  *out_row_ctr += num_rows;
  upsample->rows_to_go -= num_rows;
  /* When the buffer is emptied, declare this input row group consumed */
  if (!upsample->spare_full)
    (*in_row_group_ctr)++;
}


h2v2_merged_upsample(j_decompress_ptr cinfo, JSAMPIMAGE input_buf,
                     JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf)
{
  switch (cinfo->out_color_space) {
  case JCS_EXT_RGB:
    extrgb_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                         output_buf);
    break;
  case JCS_EXT_RGBX:
  case JCS_EXT_RGBA:
    extrgbx_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                          output_buf);
    break;
  case JCS_EXT_BGR:
    extbgr_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                         output_buf);
    break;
  case JCS_EXT_BGRX:
  case JCS_EXT_BGRA:
    extbgrx_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                          output_buf);
    break;
  case JCS_EXT_XBGR:
  case JCS_EXT_ABGR:
    extxbgr_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                          output_buf);
    break;
  case JCS_EXT_XRGB:
  case JCS_EXT_ARGB:
    extxrgb_h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                          output_buf);
    break;
  default:
    h2v2_merged_upsample_internal(cinfo, input_buf, in_row_group_ctr,
                                  output_buf);
    break;
  }
}


static int  __init i740fb_setup(char *options)
{
	char *opt;

	if (!options || !*options)
		return 0;

	while ((opt = strsep(&options, ",")) != NULL) {
		if (!*opt)
			continue;
		else if (!strncmp(opt, "mtrr:", 5))
			mtrr = simple_strtoul(opt + 5, NULL, 0);
		else
			mode_option = opt;
	}

	return 0;
}


static u32 i740_calc_fifo(struct i740fb_par *par, u32 freq, int bpp)
{
	/*
	 * Would like to calculate these values automatically, but a generic
	 * algorithm does not seem possible.  Note: These FIFO water mark
	 * values were tested on several cards and seem to eliminate the
	 * all of the snow and vertical banding, but fine adjustments will
	 * probably be required for other cards.
	 */

	u32 wm;

	switch (bpp) {
	case 8:
		if	(freq > 200)
			wm = 0x18120000;
		else if (freq > 175)
			wm = 0x16110000;
		else if (freq > 135)
			wm = 0x120E0000;
		else
			wm = 0x100D0000;
		break;
	case 15:
	case 16:
		if (par->has_sgram) {
			if	(freq > 140)
				wm = 0x2C1D0000;
			else if (freq > 120)
				wm = 0x2C180000;
			else if (freq > 100)
				wm = 0x24160000;
			else if (freq >  90)
				wm = 0x18120000;
			else if (freq >  50)
				wm = 0x16110000;
			else if (freq >  32)
				wm = 0x13100000;
			else
				wm = 0x120E0000;
		} else {
			if	(freq > 160)
				wm = 0x28200000;
			else if (freq > 140)
				wm = 0x2A1E0000;
			else if (freq > 130)
				wm = 0x2B1A0000;
			else if (freq > 120)
				wm = 0x2C180000;
			else if (freq > 100)
				wm = 0x24180000;
			else if (freq >  90)
				wm = 0x18120000;
			else if (freq >  50)
				wm = 0x16110000;
			else if (freq >  32)
				wm = 0x13100000;
			else
				wm = 0x120E0000;
		}
		break;
	case 24:
		if (par->has_sgram) {
			if	(freq > 130)
				wm = 0x31200000;
			else if (freq > 120)
				wm = 0x2E200000;
			else if (freq > 100)
				wm = 0x2C1D0000;
			else if (freq >  80)
				wm = 0x25180000;
			else if (freq >  64)
				wm = 0x24160000;
			else if (freq >  49)
				wm = 0x18120000;
			else if (freq >  32)
				wm = 0x16110000;
			else
				wm = 0x13100000;
		} else {
			if	(freq > 120)
				wm = 0x311F0000;
			else if (freq > 100)
				wm = 0x2C1D0000;
			else if (freq >  80)
				wm = 0x25180000;
			else if (freq >  64)
				wm = 0x24160000;
			else if (freq >  49)
				wm = 0x18120000;
			else if (freq >  32)
				wm = 0x16110000;
			else
				wm = 0x13100000;
		}
		break;
	case 32:
		if (par->has_sgram) {
			if	(freq >  80)
				wm = 0x2A200000;
			else if (freq >  60)
				wm = 0x281A0000;
			else if (freq >  49)
				wm = 0x25180000;
			else if (freq >  32)
				wm = 0x18120000;
			else
				wm = 0x16110000;
		} else {
			if	(freq >  80)
				wm = 0x29200000;
			else if (freq >  60)
				wm = 0x281A0000;
			else if (freq >  49)
				wm = 0x25180000;
			else if (freq >  32)
				wm = 0x18120000;
			else
				wm = 0x16110000;
		}
		break;
	}

	return wm;
}


static void vga_unprotect(struct i740fb_par *par)
{
	/* reenable display */
	i740outreg_mask(par, VGA_SEQ_I, VGA_SEQ_CLOCK_MODE, 0, 0x20);

	i740inb(par, 0x3DA);
	i740outb(par, VGA_ATT_W, 0x20);	/* disable palette access */
}


static int i740fb_setup_ddc_bus(struct fb_info *info)
{
	struct i740fb_par *par = info->par;

	strlcpy(par->ddc_adapter.name, info->fix.id,
		sizeof(par->ddc_adapter.name));
	par->ddc_adapter.owner		= THIS_MODULE;
	par->ddc_adapter.class		= I2C_CLASS_DDC;
	par->ddc_adapter.algo_data	= &par->ddc_algo;
	par->ddc_adapter.dev.parent	= info->device;
	par->ddc_algo.setsda		= i740fb_ddc_setsda;
	par->ddc_algo.setscl		= i740fb_ddc_setscl;
	par->ddc_algo.getsda		= i740fb_ddc_getsda;
	par->ddc_algo.getscl		= i740fb_ddc_getscl;
	par->ddc_algo.udelay		= 10;
	par->ddc_algo.timeout		= 20;
	par->ddc_algo.data		= par;

	i2c_set_adapdata(&par->ddc_adapter, par);

	return i2c_bit_add_bus(&par->ddc_adapter);
}


mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec)
{
	const uint16_t cqe_n = 1 << rxq->cqe_n;
	const uint16_t cqe_mask = cqe_n - 1;
	const uint16_t wqe_n = 1 << rxq->elts_n;
	const uint16_t strd_n = RTE_BIT32(rxq->log_strd_num);
	struct mlx5_rxq_ctrl *rxq_ctrl =
			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
	union {
		volatile struct mlx5_cqe *cqe;
		volatile struct mlx5_err_cqe *err_cqe;
	} u = {
		.cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask],
	};
	struct mlx5_mp_arg_queue_state_modify sm;
	int ret;

	switch (rxq->err_state) {
	case MLX5_RXQ_ERR_STATE_NO_ERROR:
		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET;
		/* Fall-through */
	case MLX5_RXQ_ERR_STATE_NEED_RESET:
		sm.is_wq = 1;
		sm.queue_id = rxq->idx;
		sm.state = IBV_WQS_RESET;
		if (mlx5_queue_state_modify(RXQ_DEV(rxq_ctrl), &sm))
			return MLX5_RECOVERY_ERROR_RET;
		if (rxq_ctrl->dump_file_n <
		    RXQ_PORT(rxq_ctrl)->config.max_dump_files_num) {
			MKSTR(err_str, "Unexpected CQE error syndrome "
			      "0x%02x CQN = %u RQN = %u wqe_counter = %u"
			      " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome,
			      rxq->cqn, rxq_ctrl->wqn,
			      rte_be_to_cpu_16(u.err_cqe->wqe_counter),
			      rxq->rq_ci << rxq->sges_n, rxq->cq_ci);
			MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u",
			      rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc());
			mlx5_dump_debug_information(name, NULL, err_str, 0);
			mlx5_dump_debug_information(name, "MLX5 Error CQ:",
						    (const void *)((uintptr_t)
								    rxq->cqes),
						    sizeof(*u.cqe) * cqe_n);
			mlx5_dump_debug_information(name, "MLX5 Error RQ:",
						    (const void *)((uintptr_t)
								    rxq->wqes),
						    16 * wqe_n);
			rxq_ctrl->dump_file_n++;
		}
		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY;
		/* Fall-through */
	case MLX5_RXQ_ERR_STATE_NEED_READY:
		ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci);
		if (ret == MLX5_CQE_STATUS_HW_OWN) {
			rte_io_wmb();
			*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
			rte_io_wmb();
			/*
			 * The RQ consumer index must be zeroed while moving
			 * from RESET state to RDY state.
			 */
			*rxq->rq_db = rte_cpu_to_be_32(0);
			rte_io_wmb();
			sm.is_wq = 1;
			sm.queue_id = rxq->idx;
			sm.state = IBV_WQS_RDY;
			if (mlx5_queue_state_modify(RXQ_DEV(rxq_ctrl), &sm))
				return MLX5_RECOVERY_ERROR_RET;
			if (vec) {
				const uint32_t elts_n =
					mlx5_rxq_mprq_enabled(rxq) ?
					wqe_n * strd_n : wqe_n;
				const uint32_t e_mask = elts_n - 1;
				uint32_t elts_ci =
					mlx5_rxq_mprq_enabled(rxq) ?
					rxq->elts_ci : rxq->rq_ci;
				uint32_t elt_idx;
				struct rte_mbuf **elt;
				int i;
				unsigned int n = elts_n - (elts_ci -
							  rxq->rq_pi);

				for (i = 0; i < (int)n; ++i) {
					elt_idx = (elts_ci + i) & e_mask;
					elt = &(*rxq->elts)[elt_idx];
					*elt = rte_mbuf_raw_alloc(rxq->mp);
					if (!*elt) {
						for (i--; i >= 0; --i) {
							elt_idx = (elts_ci +
								   i) & elts_n;
							elt = &(*rxq->elts)
								[elt_idx];
							rte_pktmbuf_free_seg
								(*elt);
						}
						return MLX5_RECOVERY_ERROR_RET;
					}
				}
				for (i = 0; i < (int)elts_n; ++i) {
					elt = &(*rxq->elts)[i];
					DATA_LEN(*elt) =
						(uint16_t)((*elt)->buf_len -
						rte_pktmbuf_headroom(*elt));
				}
				/* Padding with a fake mbuf for vec Rx. */
				for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
					(*rxq->elts)[elts_n + i] =
								&rxq->fake_mbuf;
			}
			mlx5_rxq_initialize(rxq);
			rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR;
		}
		return ret;
	default:
		return MLX5_RECOVERY_ERROR_RET;
	}
}


mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
		  struct rte_eth_rxq_info *qinfo)
{
	struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, rx_queue_id);
	struct mlx5_rxq_data *rxq = mlx5_rxq_data_get(dev, rx_queue_id);
	struct mlx5_rxq_priv *rxq_priv = mlx5_rxq_get(dev, rx_queue_id);

	if (!rxq)
		return;
	qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
					rxq->mprq_mp : rxq->mp;
	qinfo->conf.rx_thresh.pthresh = 0;
	qinfo->conf.rx_thresh.hthresh = 0;
	qinfo->conf.rx_thresh.wthresh = 0;
	qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
	qinfo->conf.rx_drop_en = 1;
	if (rxq_ctrl == NULL || rxq_ctrl->obj == NULL)
		qinfo->conf.rx_deferred_start = 0;
	else
		qinfo->conf.rx_deferred_start = 1;
	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
	qinfo->scattered_rx = dev->data->scattered_rx;
	qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
		RTE_BIT32(rxq->elts_n) * RTE_BIT32(rxq->log_strd_num) :
		RTE_BIT32(rxq->elts_n);
	qinfo->avail_thresh = rxq_priv ?
		mlx5_rxq_lwm_to_percentage(rxq_priv) : 0;
}


mlx5_tx_match_empw(struct mlx5_txq_data *__rte_restrict txq,
		   struct mlx5_wqe_eseg *__rte_restrict es,
		   struct mlx5_txq_local *__rte_restrict loc,
		   uint32_t dlen,
		   unsigned int olx)
{
	uint8_t swp_flags = 0;

	/* Compare the checksum flags, if any. */
	if (MLX5_TXOFF_CONFIG(CSUM) &&
	    txq_ol_cksum_to_cs(loc->mbuf) != es->cs_flags)
		return false;
	/* Compare the Software Parser offsets and flags. */
	if (MLX5_TXOFF_CONFIG(SWP) &&
	    (es->swp_offs != txq_mbuf_to_swp(loc, &swp_flags, olx) ||
	     es->swp_flags != swp_flags))
		return false;
	/* Fill metadata field if needed. */
	if (MLX5_TXOFF_CONFIG(METADATA) &&
		es->metadata != (loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ?
				 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0))
		return false;
	/* Legacy MPW can send packets with the same lengt only. */
	if (MLX5_TXOFF_CONFIG(MPW) &&
	    dlen != rte_pktmbuf_data_len(loc->mbuf))
		return false;
	/* There must be no VLAN packets in eMPW loop. */
	if (MLX5_TXOFF_CONFIG(VLAN))
		MLX5_ASSERT(!(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT));
	/* Check if the scheduling is requested. */
	if (MLX5_TXOFF_CONFIG(TXPP) &&
	    loc->mbuf->ol_flags & txq->ts_mask)
		return false;
	return true;
}


mlx5_tx_burst_empw_simple(struct mlx5_txq_data *__rte_restrict txq,
			  struct rte_mbuf **__rte_restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *__rte_restrict loc,
			  unsigned int olx)
{
	/*
	 * Subroutine is the part of mlx5_tx_burst_single()
	 * and sends single-segment packet with eMPW opcode
	 * without data inlining.
	 */
	MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE));
	MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
	MLX5_ASSERT(loc->elts_free && loc->wqe_free);
	MLX5_ASSERT(pkts_n > loc->pkts_sent);
	static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe_dseg *__rte_restrict dseg;
		struct mlx5_wqe_eseg *__rte_restrict eseg;
		enum mlx5_txcmp_code ret;
		unsigned int part, loop;
		unsigned int slen = 0;

next_empw:
		MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
		if (MLX5_TXOFF_CONFIG(TXPP)) {
			enum mlx5_txcmp_code wret;

			/* Generate WAIT for scheduling if requested. */
			wret = mlx5_tx_schedule_send(txq, loc, olx);
			if (wret == MLX5_TXCMP_CODE_EXIT)
				return MLX5_TXCMP_CODE_EXIT;
			if (wret == MLX5_TXCMP_CODE_ERROR)
				return MLX5_TXCMP_CODE_ERROR;
		}
		part = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
				       MLX5_MPW_MAX_PACKETS :
				       MLX5_EMPW_MAX_PACKETS);
		if (unlikely(loc->elts_free < part)) {
			/* We have no enough elts to save all mbufs. */
			if (unlikely(loc->elts_free < MLX5_EMPW_MIN_PACKETS))
				return MLX5_TXCMP_CODE_EXIT;
			/* But we still able to send at least minimal eMPW. */
			part = loc->elts_free;
		}
		/* Check whether we have enough WQEs */
		if (unlikely(loc->wqe_free < ((2 + part + 3) / 4))) {
			if (unlikely(loc->wqe_free <
				((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
				return MLX5_TXCMP_CODE_EXIT;
			part = (loc->wqe_free * 4) - 2;
		}
		if (likely(part > 1))
			rte_prefetch0(*pkts);
		loc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m);
		/*
		 * Build eMPW title WQEBB:
		 * - Control Segment, eMPW opcode
		 * - Ethernet Segment, no inline
		 */
		mlx5_tx_cseg_init(txq, loc, loc->wqe_last, part + 2,
				  MLX5_OPCODE_ENHANCED_MPSW, olx);
		mlx5_tx_eseg_none(txq, loc, loc->wqe_last,
				  olx & ~MLX5_TXOFF_CONFIG_VLAN);
		eseg = &loc->wqe_last->eseg;
		dseg = &loc->wqe_last->dseg[0];
		loop = part;
		/* Store the packet length for legacy MPW. */
		if (MLX5_TXOFF_CONFIG(MPW))
			eseg->mss = rte_cpu_to_be_16
					(rte_pktmbuf_data_len(loc->mbuf));
		for (;;) {
			uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
#ifdef MLX5_PMD_SOFT_COUNTERS
			/* Update sent data bytes counter. */
			slen += dlen;
#endif
			mlx5_tx_dseg_ptr
				(txq, loc, dseg,
				 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
				 dlen, olx);
			if (unlikely(--loop == 0))
				break;
			loc->mbuf = *pkts++;
			if (likely(loop > 1))
				rte_prefetch0(*pkts);
			ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
			/*
			 * Unroll the completion code to avoid
			 * returning variable value - it results in
			 * unoptimized sequent checking in caller.
			 */
			if (ret == MLX5_TXCMP_CODE_MULTI) {
				part -= loop;
				mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_MULTI;
			}
			MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
			if (ret == MLX5_TXCMP_CODE_TSO) {
				part -= loop;
				mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_TSO;
			}
			if (ret == MLX5_TXCMP_CODE_SINGLE) {
				part -= loop;
				mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_SINGLE;
			}
			if (ret != MLX5_TXCMP_CODE_EMPW) {
				MLX5_ASSERT(false);
				part -= loop;
				mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			/*
			 * Check whether packet parameters coincide
			 * within assumed eMPW batch:
			 * - check sum settings
			 * - metadata value
			 * - software parser settings
			 * - packets length (legacy MPW only)
			 * - scheduling is not required
			 */
			if (!mlx5_tx_match_empw(txq, eseg, loc, dlen, olx)) {
				MLX5_ASSERT(loop);
				part -= loop;
				mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				pkts_n -= part;
				goto next_empw;
			}
			/* Packet attributes match, continue the same eMPW. */
			++dseg;
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
		}
		/* eMPW is built successfully, update loop parameters. */
		MLX5_ASSERT(!loop);
		MLX5_ASSERT(pkts_n >= part);
#ifdef MLX5_PMD_SOFT_COUNTERS
		/* Update sent data bytes counter. */
		txq->stats.obytes += slen;
#endif
		loc->elts_free -= part;
		loc->pkts_sent += part;
		txq->wqe_ci += (2 + part + 3) / 4;
		loc->wqe_free -= (2 + part + 3) / 4;
		pkts_n -= part;
		if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		loc->mbuf = *pkts++;
		ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
		if (unlikely(ret != MLX5_TXCMP_CODE_EMPW))
			return ret;
		/* Continue sending eMPW batches. */
	}
	MLX5_ASSERT(false);
}


rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
{
	uint32_t ol_flags = 0;
	uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);

	ol_flags =
		TRANSPOSE(flags,
			  MLX5_CQE_RX_L3_HDR_VALID,
			  PKT_RX_IP_CKSUM_GOOD) |
		TRANSPOSE(flags,
			  MLX5_CQE_RX_L4_HDR_VALID,
			  PKT_RX_L4_CKSUM_GOOD);
	return ol_flags;
}


mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_rxq_data *rxq;

	if (dev->rx_pkt_burst == NULL ||
	    dev->rx_pkt_burst == removed_rx_burst) {
		rte_errno = ENOTSUP;
		return -rte_errno;
	}
	rxq = (*priv->rxqs)[rx_queue_id];
	if (!rxq) {
		rte_errno = EINVAL;
		return -rte_errno;
	}
	return rx_queue_count(rxq);
}


mlx5_tx_mseg_memcpy(uint8_t *pdst,
		    struct mlx5_txq_local *__rte_restrict loc,
		    unsigned int len,
		    unsigned int must,
		    unsigned int olx __rte_unused)
{
	struct rte_mbuf *mbuf;
	unsigned int part, dlen, copy = 0;
	uint8_t *psrc;

	MLX5_ASSERT(len);
	do {
		/* Allow zero length packets, must check first. */
		dlen = rte_pktmbuf_data_len(loc->mbuf);
		if (dlen <= loc->mbuf_off) {
			/* Exhausted packet, just free. */
			mbuf = loc->mbuf;
			loc->mbuf = mbuf->next;
			rte_pktmbuf_free_seg(mbuf);
			loc->mbuf_off = 0;
			MLX5_ASSERT(loc->mbuf_nseg > 1);
			MLX5_ASSERT(loc->mbuf);
			--loc->mbuf_nseg;
			if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) {
				unsigned int diff;

				if (copy >= must) {
					/*
					 * We already copied the minimal
					 * requested amount of data.
					 */
					return copy;
				}
				diff = must - copy;
				if (diff <= rte_pktmbuf_data_len(loc->mbuf)) {
					/*
					 * Copy only the minimal required
					 * part of the data buffer. Limit amount
					 * of data to be copied to the length of
					 * available space.
					 */
					len = RTE_MIN(len, diff);
				}
			}
			continue;
		}
		dlen -= loc->mbuf_off;
		psrc = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *,
					       loc->mbuf_off);
		part = RTE_MIN(len, dlen);
		rte_memcpy(pdst, psrc, part);
		copy += part;
		loc->mbuf_off += part;
		len -= part;
		if (!len) {
			if (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) {
				loc->mbuf_off = 0;
				/* Exhausted packet, just free. */
				mbuf = loc->mbuf;
				loc->mbuf = mbuf->next;
				rte_pktmbuf_free_seg(mbuf);
				loc->mbuf_off = 0;
				MLX5_ASSERT(loc->mbuf_nseg >= 1);
				--loc->mbuf_nseg;
			}
			return copy;
		}
		pdst += part;
	} while (true);
}


mlx5_tx_mseg_build(struct mlx5_txq_data *__rte_restrict txq,
		   struct mlx5_txq_local *__rte_restrict loc,
		   struct mlx5_wqe *__rte_restrict wqe,
		   unsigned int vlan,
		   unsigned int inlen,
		   unsigned int tso,
		   unsigned int olx __rte_unused)
{
	struct mlx5_wqe_dseg *__rte_restrict dseg;
	unsigned int ds;

	MLX5_ASSERT((rte_pktmbuf_pkt_len(loc->mbuf) + vlan) >= inlen);
	loc->mbuf_nseg = NB_SEGS(loc->mbuf);
	loc->mbuf_off = 0;

	dseg = mlx5_tx_eseg_mdat(txq, loc, wqe, vlan, inlen, tso, olx);
	if (!loc->mbuf_nseg)
		goto dseg_done;
	/*
	 * There are still some mbuf remaining, not inlined.
	 * The first mbuf may be partially inlined and we
	 * must process the possible non-zero data offset.
	 */
	if (loc->mbuf_off) {
		unsigned int dlen;
		uint8_t *dptr;

		/*
		 * Exhausted packets must be dropped before.
		 * Non-zero offset means there are some data
		 * remained in the packet.
		 */
		MLX5_ASSERT(loc->mbuf_off < rte_pktmbuf_data_len(loc->mbuf));
		MLX5_ASSERT(rte_pktmbuf_data_len(loc->mbuf));
		dptr = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *,
					       loc->mbuf_off);
		dlen = rte_pktmbuf_data_len(loc->mbuf) - loc->mbuf_off;
		/*
		 * Build the pointer/minimal data Data Segment.
		 * Do ring buffer wrapping check in advance.
		 */
		if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
			dseg = (struct mlx5_wqe_dseg *)txq->wqes;
		mlx5_tx_dseg_iptr(txq, loc, dseg, dptr, dlen, olx);
		/* Store the mbuf to be freed on completion. */
		MLX5_ASSERT(loc->elts_free);
		txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
		--loc->elts_free;
		++dseg;
		if (--loc->mbuf_nseg == 0)
			goto dseg_done;
		loc->mbuf = loc->mbuf->next;
		loc->mbuf_off = 0;
	}
	do {
		if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {
			struct rte_mbuf *mbuf;

			/* Zero length segment found, just skip. */
			mbuf = loc->mbuf;
			loc->mbuf = loc->mbuf->next;
			rte_pktmbuf_free_seg(mbuf);
			if (--loc->mbuf_nseg == 0)
				break;
		} else {
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
			mlx5_tx_dseg_iptr
				(txq, loc, dseg,
				 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
				 rte_pktmbuf_data_len(loc->mbuf), olx);
			MLX5_ASSERT(loc->elts_free);
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
			--loc->elts_free;
			++dseg;
			if (--loc->mbuf_nseg == 0)
				break;
			loc->mbuf = loc->mbuf->next;
		}
	} while (true);

dseg_done:
	/* Calculate actual segments used from the dseg pointer. */
	if ((uintptr_t)wqe < (uintptr_t)dseg)
		ds = ((uintptr_t)dseg - (uintptr_t)wqe) / MLX5_WSEG_SIZE;
	else
		ds = (((uintptr_t)dseg - (uintptr_t)wqe) +
		      txq->wqe_s * MLX5_WQE_SIZE) / MLX5_WSEG_SIZE;
	return ds;
}


mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
		  struct rte_eth_txq_info *qinfo)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id];
	struct mlx5_txq_ctrl *txq_ctrl =
			container_of(txq, struct mlx5_txq_ctrl, txq);

	if (!txq)
		return;
	qinfo->nb_desc = txq->elts_s;
	qinfo->conf.tx_thresh.pthresh = 0;
	qinfo->conf.tx_thresh.hthresh = 0;
	qinfo->conf.tx_thresh.wthresh = 0;
	qinfo->conf.tx_rs_thresh = 0;
	qinfo->conf.tx_free_thresh = 0;
	qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1;
	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
}


mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
		  struct rte_eth_rxq_info *qinfo)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id];
	struct mlx5_rxq_ctrl *rxq_ctrl =
		container_of(rxq, struct mlx5_rxq_ctrl, rxq);

	if (!rxq)
		return;
	qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
					rxq->mprq_mp : rxq->mp;
	qinfo->conf.rx_thresh.pthresh = 0;
	qinfo->conf.rx_thresh.hthresh = 0;
	qinfo->conf.rx_thresh.wthresh = 0;
	qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
	qinfo->conf.rx_drop_en = 1;
	qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
	qinfo->scattered_rx = dev->data->scattered_rx;
	qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
		RTE_BIT32(rxq->elts_n) * RTE_BIT32(rxq->log_strd_num) :
		RTE_BIT32(rxq->elts_n);
}


mlx5_tx_dseg_ptr(struct mlx5_txq_data *__rte_restrict txq,
		 struct mlx5_txq_local *__rte_restrict loc,
		 struct mlx5_wqe_dseg *__rte_restrict dseg,
		 uint8_t *buf,
		 unsigned int len,
		 unsigned int olx __rte_unused)

{
	MLX5_ASSERT(len);
	dseg->bcount = rte_cpu_to_be_32(len);
	dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);
	dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);
}


mlx5_tx_cseg_init(struct mlx5_txq_data *restrict txq,
		  struct mlx5_txq_local *restrict loc __rte_unused,
		  struct mlx5_wqe *restrict wqe,
		  unsigned int ds,
		  unsigned int opcode,
		  unsigned int olx __rte_unused)
{
	struct mlx5_wqe_cseg *restrict cs = &wqe->cseg;

	/* For legacy MPW replace the EMPW by TSO with modifier. */
	if (MLX5_TXOFF_CONFIG(MPW) && opcode == MLX5_OPCODE_ENHANCED_MPSW)
		opcode = MLX5_OPCODE_TSO | MLX5_OPC_MOD_MPW << 24;
	cs->opcode = rte_cpu_to_be_32((txq->wqe_ci << 8) | opcode);
	cs->sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
	cs->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
			     MLX5_COMP_MODE_OFFSET);
	cs->misc = RTE_BE32(0);
}


mlx5_tx_dseg_vlan(struct mlx5_txq_data *restrict txq,
		  struct mlx5_txq_local *restrict loc __rte_unused,
		  struct mlx5_wqe_dseg *restrict dseg,
		  uint8_t *buf,
		  unsigned int len,
		  unsigned int olx __rte_unused)

{
	unsigned int part;
	uint8_t *pdst;

	assert(len > MLX5_ESEG_MIN_INLINE_SIZE);
	static_assert(MLX5_DSEG_MIN_INLINE_SIZE ==
				 (2 * RTE_ETHER_ADDR_LEN),
		      "invalid Data Segment data size");
	if (!MLX5_TXOFF_CONFIG(MPW)) {
		/* Store the descriptor byte counter for eMPW sessions. */
		dseg->bcount = rte_cpu_to_be_32
				((len + sizeof(struct rte_vlan_hdr)) |
				 MLX5_ETH_WQE_DATA_INLINE);
		pdst = &dseg->inline_data[0];
	} else {
		/* The entire legacy MPW session counter is stored on close. */
		pdst = (uint8_t *)dseg;
	}
	memcpy(pdst, buf, MLX5_DSEG_MIN_INLINE_SIZE);
	buf += MLX5_DSEG_MIN_INLINE_SIZE;
	pdst += MLX5_DSEG_MIN_INLINE_SIZE;
	len -= MLX5_DSEG_MIN_INLINE_SIZE;
	/* Insert VLAN ethertype + VLAN tag. Pointer is aligned. */
	assert(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE));
	if (unlikely(pdst >= (uint8_t *)txq->wqes_end))
		pdst = (uint8_t *)txq->wqes;
	*(uint32_t *)pdst = rte_cpu_to_be_32((RTE_ETHER_TYPE_VLAN << 16) |
					      loc->mbuf->vlan_tci);
	pdst += sizeof(struct rte_vlan_hdr);
	/*
	 * The WQEBB space availability is checked by caller.
	 * Here we should be aware of WQE ring buffer wraparound only.
	 */
	part = (uint8_t *)txq->wqes_end - pdst;
	part = RTE_MIN(part, len);
	do {
		rte_memcpy(pdst, buf, part);
		len -= part;
		if (likely(!len)) {
			pdst += part;
			if (!MLX5_TXOFF_CONFIG(MPW))
				pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);
			/* Note: no final wraparound check here. */
			return (struct mlx5_wqe_dseg *)pdst;
		}
		pdst = (uint8_t *)txq->wqes;
		buf += part;
		part = len;
	} while (true);
}


static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
		       struct packet_type *pt, struct net_device *orig_dev)
{
	struct sock *sk;
	struct packet_sock *po;
	struct sockaddr_ll *sll;
	union tpacket_uhdr h;
	u8 *skb_head = skb->data;
	int skb_len = skb->len;
	unsigned int snaplen, res;
	unsigned long status = TP_STATUS_USER;
	unsigned short macoff, hdrlen;
	unsigned int netoff;
	struct sk_buff *copy_skb = NULL;
	struct timespec64 ts;
	__u32 ts_status;
	bool is_drop_n_account = false;
	unsigned int slot_id = 0;
	bool do_vnet = false;

	/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
	 * We may add members to them until current aligned size without forcing
	 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
	 */
	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);

	if (skb->pkt_type == PACKET_LOOPBACK)
		goto drop;

	sk = pt->af_packet_priv;
	po = pkt_sk(sk);

	if (!net_eq(dev_net(dev), sock_net(sk)))
		goto drop;

	if (dev_has_header(dev)) {
		if (sk->sk_type != SOCK_DGRAM)
			skb_push(skb, skb->data - skb_mac_header(skb));
		else if (skb->pkt_type == PACKET_OUTGOING) {
			/* Special case: outgoing packets have ll header at head */
			skb_pull(skb, skb_network_offset(skb));
		}
	}

	snaplen = skb->len;

	res = run_filter(skb, sk, snaplen);
	if (!res)
		goto drop_n_restore;

	/* If we are flooded, just give up */
	if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
		atomic_inc(&po->tp_drops);
		goto drop_n_restore;
	}

	if (skb->ip_summed == CHECKSUM_PARTIAL)
		status |= TP_STATUS_CSUMNOTREADY;
	else if (skb->pkt_type != PACKET_OUTGOING &&
		 (skb->ip_summed == CHECKSUM_COMPLETE ||
		  skb_csum_unnecessary(skb)))
		status |= TP_STATUS_CSUM_VALID;

	if (snaplen > res)
		snaplen = res;

	if (sk->sk_type == SOCK_DGRAM) {
		macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
				  po->tp_reserve;
	} else {
		unsigned int maclen = skb_network_offset(skb);
		netoff = TPACKET_ALIGN(po->tp_hdrlen +
				       (maclen < 16 ? 16 : maclen)) +
				       po->tp_reserve;
		if (po->has_vnet_hdr) {
			netoff += sizeof(struct virtio_net_hdr);
			do_vnet = true;
		}
		macoff = netoff - maclen;
	}
	if (netoff > USHRT_MAX) {
		atomic_inc(&po->tp_drops);
		goto drop_n_restore;
	}
	if (po->tp_version <= TPACKET_V2) {
		if (macoff + snaplen > po->rx_ring.frame_size) {
			if (po->copy_thresh &&
			    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
				if (skb_shared(skb)) {
					copy_skb = skb_clone(skb, GFP_ATOMIC);
				} else {
					copy_skb = skb_get(skb);
					skb_head = skb->data;
				}
				if (copy_skb) {
					memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
					       sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
					skb_set_owner_r(copy_skb, sk);
				}
			}
			snaplen = po->rx_ring.frame_size - macoff;
			if ((int)snaplen < 0) {
				snaplen = 0;
				do_vnet = false;
			}
		}
	} else if (unlikely(macoff + snaplen >
			    GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
		u32 nval;

		nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
		pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
			    snaplen, nval, macoff);
		snaplen = nval;
		if (unlikely((int)snaplen < 0)) {
			snaplen = 0;
			macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
			do_vnet = false;
		}
	}
	spin_lock(&sk->sk_receive_queue.lock);
	h.raw = packet_current_rx_frame(po, skb,
					TP_STATUS_KERNEL, (macoff+snaplen));
	if (!h.raw)
		goto drop_n_account;

	if (po->tp_version <= TPACKET_V2) {
		slot_id = po->rx_ring.head;
		if (test_bit(slot_id, po->rx_ring.rx_owner_map))
			goto drop_n_account;
		__set_bit(slot_id, po->rx_ring.rx_owner_map);
	}

	if (do_vnet &&
	    virtio_net_hdr_from_skb(skb, h.raw + macoff -
				    sizeof(struct virtio_net_hdr),
				    vio_le(), true, 0)) {
		if (po->tp_version == TPACKET_V3)
			prb_clear_blk_fill_status(&po->rx_ring);
		goto drop_n_account;
	}

	if (po->tp_version <= TPACKET_V2) {
		packet_increment_rx_head(po, &po->rx_ring);
	/*
	 * LOSING will be reported till you read the stats,
	 * because it's COR - Clear On Read.
	 * Anyways, moving it for V1/V2 only as V3 doesn't need this
	 * at packet level.
	 */
		if (atomic_read(&po->tp_drops))
			status |= TP_STATUS_LOSING;
	}

	po->stats.stats1.tp_packets++;
	if (copy_skb) {
		status |= TP_STATUS_COPY;
		__skb_queue_tail(&sk->sk_receive_queue, copy_skb);
	}
	spin_unlock(&sk->sk_receive_queue.lock);

	skb_copy_bits(skb, 0, h.raw + macoff, snaplen);

	/* Always timestamp; prefer an existing software timestamp taken
	 * closer to the time of capture.
	 */
	ts_status = tpacket_get_timestamp(skb, &ts,
					  po->tp_tstamp | SOF_TIMESTAMPING_SOFTWARE);
	if (!ts_status)
		ktime_get_real_ts64(&ts);

	status |= ts_status;

	switch (po->tp_version) {
	case TPACKET_V1:
		h.h1->tp_len = skb->len;
		h.h1->tp_snaplen = snaplen;
		h.h1->tp_mac = macoff;
		h.h1->tp_net = netoff;
		h.h1->tp_sec = ts.tv_sec;
		h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
		hdrlen = sizeof(*h.h1);
		break;
	case TPACKET_V2:
		h.h2->tp_len = skb->len;
		h.h2->tp_snaplen = snaplen;
		h.h2->tp_mac = macoff;
		h.h2->tp_net = netoff;
		h.h2->tp_sec = ts.tv_sec;
		h.h2->tp_nsec = ts.tv_nsec;
		if (skb_vlan_tag_present(skb)) {
			h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
			h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
			status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
		} else {
			h.h2->tp_vlan_tci = 0;
			h.h2->tp_vlan_tpid = 0;
		}
		memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
		hdrlen = sizeof(*h.h2);
		break;
	case TPACKET_V3:
		/* tp_nxt_offset,vlan are already populated above.
		 * So DONT clear those fields here
		 */
		h.h3->tp_status |= status;
		h.h3->tp_len = skb->len;
		h.h3->tp_snaplen = snaplen;
		h.h3->tp_mac = macoff;
		h.h3->tp_net = netoff;
		h.h3->tp_sec  = ts.tv_sec;
		h.h3->tp_nsec = ts.tv_nsec;
		memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
		hdrlen = sizeof(*h.h3);
		break;
	default:
		BUG();
	}

	sll = h.raw + TPACKET_ALIGN(hdrlen);
	sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
	sll->sll_family = AF_PACKET;
	sll->sll_hatype = dev->type;
	sll->sll_protocol = skb->protocol;
	sll->sll_pkttype = skb->pkt_type;
	if (unlikely(po->origdev))
		sll->sll_ifindex = orig_dev->ifindex;
	else
		sll->sll_ifindex = dev->ifindex;

	smp_mb();

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
	if (po->tp_version <= TPACKET_V2) {
		u8 *start, *end;

		end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
					macoff + snaplen);

		for (start = h.raw; start < end; start += PAGE_SIZE)
			flush_dcache_page(pgv_to_page(start));
	}
	smp_wmb();
#endif

	if (po->tp_version <= TPACKET_V2) {
		spin_lock(&sk->sk_receive_queue.lock);
		__packet_set_status(po, h.raw, status);
		__clear_bit(slot_id, po->rx_ring.rx_owner_map);
		spin_unlock(&sk->sk_receive_queue.lock);
		sk->sk_data_ready(sk);
	} else if (po->tp_version == TPACKET_V3) {
		prb_clear_blk_fill_status(&po->rx_ring);
	}

drop_n_restore:
	if (skb_head != skb->data && skb_shared(skb)) {
		skb->data = skb_head;
		skb->len = skb_len;
	}
drop:
	if (!is_drop_n_account)
		consume_skb(skb);
	else
		kfree_skb(skb);
	return 0;

drop_n_account:
	spin_unlock(&sk->sk_receive_queue.lock);
	atomic_inc(&po->tp_drops);
	is_drop_n_account = true;

	sk->sk_data_ready(sk);
	kfree_skb(copy_skb);
	goto drop_n_restore;
}


static int fanout_add(struct sock *sk, struct fanout_args *args)
{
	struct packet_rollover *rollover = NULL;
	struct packet_sock *po = pkt_sk(sk);
	u16 type_flags = args->type_flags;
	struct packet_fanout *f, *match;
	u8 type = type_flags & 0xff;
	u8 flags = type_flags >> 8;
	u16 id = args->id;
	int err;

	switch (type) {
	case PACKET_FANOUT_ROLLOVER:
		if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
			return -EINVAL;
		break;
	case PACKET_FANOUT_HASH:
	case PACKET_FANOUT_LB:
	case PACKET_FANOUT_CPU:
	case PACKET_FANOUT_RND:
	case PACKET_FANOUT_QM:
	case PACKET_FANOUT_CBPF:
	case PACKET_FANOUT_EBPF:
		break;
	default:
		return -EINVAL;
	}

	mutex_lock(&fanout_mutex);

	err = -EALREADY;
	if (po->fanout)
		goto out;

	if (type == PACKET_FANOUT_ROLLOVER ||
	    (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
		err = -ENOMEM;
		rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
		if (!rollover)
			goto out;
		atomic_long_set(&rollover->num, 0);
		atomic_long_set(&rollover->num_huge, 0);
		atomic_long_set(&rollover->num_failed, 0);
	}

	if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
		if (id != 0) {
			err = -EINVAL;
			goto out;
		}
		if (!fanout_find_new_id(sk, &id)) {
			err = -ENOMEM;
			goto out;
		}
		/* ephemeral flag for the first socket in the group: drop it */
		flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
	}

	match = NULL;
	list_for_each_entry(f, &fanout_list, list) {
		if (f->id == id &&
		    read_pnet(&f->net) == sock_net(sk)) {
			match = f;
			break;
		}
	}
	err = -EINVAL;
	if (match) {
		if (match->flags != flags)
			goto out;
		if (args->max_num_members &&
		    args->max_num_members != match->max_num_members)
			goto out;
	} else {
		if (args->max_num_members > PACKET_FANOUT_MAX)
			goto out;
		if (!args->max_num_members)
			/* legacy PACKET_FANOUT_MAX */
			args->max_num_members = 256;
		err = -ENOMEM;
		match = kvzalloc(struct_size(match, arr, args->max_num_members),
				 GFP_KERNEL);
		if (!match)
			goto out;
		write_pnet(&match->net, sock_net(sk));
		match->id = id;
		match->type = type;
		match->flags = flags;
		INIT_LIST_HEAD(&match->list);
		spin_lock_init(&match->lock);
		refcount_set(&match->sk_ref, 0);
		fanout_init_data(match);
		match->prot_hook.type = po->prot_hook.type;
		match->prot_hook.dev = po->prot_hook.dev;
		match->prot_hook.func = packet_rcv_fanout;
		match->prot_hook.af_packet_priv = match;
		match->prot_hook.af_packet_net = read_pnet(&match->net);
		match->prot_hook.id_match = match_fanout_group;
		match->max_num_members = args->max_num_members;
		list_add(&match->list, &fanout_list);
	}
	err = -EINVAL;

	spin_lock(&po->bind_lock);
	if (po->running &&
	    match->type == type &&
	    match->prot_hook.type == po->prot_hook.type &&
	    match->prot_hook.dev == po->prot_hook.dev) {
		err = -ENOSPC;
		if (refcount_read(&match->sk_ref) < match->max_num_members) {
			__dev_remove_pack(&po->prot_hook);

			/* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
			WRITE_ONCE(po->fanout, match);

			po->rollover = rollover;
			rollover = NULL;
			refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
			__fanout_link(sk, po);
			err = 0;
		}
	}
	spin_unlock(&po->bind_lock);

	if (err && !refcount_read(&match->sk_ref)) {
		list_del(&match->list);
		kvfree(match);
	}

out:
	kfree(rollover);
	mutex_unlock(&fanout_mutex);
	return err;
}


static int packet_create(struct net *net, struct socket *sock, int protocol,
			 int kern)
{
	struct sock *sk;
	struct packet_sock *po;
	__be16 proto = (__force __be16)protocol; /* weird, but documented */
	int err;

	if (!ns_capable(net->user_ns, CAP_NET_RAW))
		return -EPERM;
	if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
	    sock->type != SOCK_PACKET)
		return -ESOCKTNOSUPPORT;

	sock->state = SS_UNCONNECTED;

	err = -ENOBUFS;
	sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
	if (sk == NULL)
		goto out;

	sock->ops = &packet_ops;
	if (sock->type == SOCK_PACKET)
		sock->ops = &packet_ops_spkt;

	sock_init_data(sock, sk);

	po = pkt_sk(sk);
	init_completion(&po->skb_completion);
	sk->sk_family = PF_PACKET;
	po->num = proto;
	po->xmit = dev_queue_xmit;

	err = packet_alloc_pending(po);
	if (err)
		goto out2;

	packet_cached_dev_reset(po);

	sk->sk_destruct = packet_sock_destruct;
	sk_refcnt_debug_inc(sk);

	/*
	 *	Attach a protocol block
	 */

	spin_lock_init(&po->bind_lock);
	mutex_init(&po->pg_vec_lock);
	po->rollover = NULL;
	po->prot_hook.func = packet_rcv;

	if (sock->type == SOCK_PACKET)
		po->prot_hook.func = packet_rcv_spkt;

	po->prot_hook.af_packet_priv = sk;
	po->prot_hook.af_packet_net = sock_net(sk);

	if (proto) {
		po->prot_hook.type = proto;
		__register_prot_hook(sk);
	}

	mutex_lock(&net->packet.sklist_lock);
	sk_add_node_tail_rcu(sk, &net->packet.sklist);
	mutex_unlock(&net->packet.sklist_lock);

	sock_prot_inuse_add(net, &packet_proto, 1);

	return 0;
out2:
	sk_free(sk);
out:
	return err;
}


ex_splitview(exarg_T *eap)
{
    win_T	*old_curwin = curwin;
    char_u	*fname = NULL;
#ifdef FEAT_BROWSE
    char_u	dot_path[] = ".";
    int		save_cmod_flags = cmdmod.cmod_flags;
#endif
    int		use_tab = eap->cmdidx == CMD_tabedit
		       || eap->cmdidx == CMD_tabfind
		       || eap->cmdidx == CMD_tabnew;

    if (ERROR_IF_ANY_POPUP_WINDOW)
	return;

#ifdef FEAT_GUI
    need_mouse_correct = TRUE;
#endif

#ifdef FEAT_QUICKFIX
    // A ":split" in the quickfix window works like ":new".  Don't want two
    // quickfix windows.  But it's OK when doing ":tab split".
    if (bt_quickfix(curbuf) && cmdmod.cmod_tab == 0)
    {
	if (eap->cmdidx == CMD_split)
	    eap->cmdidx = CMD_new;
	if (eap->cmdidx == CMD_vsplit)
	    eap->cmdidx = CMD_vnew;
    }
#endif

    if (eap->cmdidx == CMD_sfind || eap->cmdidx == CMD_tabfind)
    {
	fname = find_file_in_path(eap->arg, (int)STRLEN(eap->arg),
					  FNAME_MESS, TRUE, curbuf->b_ffname);
	if (fname == NULL)
	    goto theend;
	eap->arg = fname;
    }
# ifdef FEAT_BROWSE
    else if ((cmdmod.cmod_flags & CMOD_BROWSE)
	    && eap->cmdidx != CMD_vnew
	    && eap->cmdidx != CMD_new)
    {
	if (
# ifdef FEAT_GUI
	    !gui.in_use &&
# endif
		au_has_group((char_u *)"FileExplorer"))
	{
	    // No browsing supported but we do have the file explorer:
	    // Edit the directory.
	    if (*eap->arg == NUL || !mch_isdir(eap->arg))
		eap->arg = dot_path;
	}
	else
	{
	    fname = do_browse(0, (char_u *)(use_tab
			? _("Edit File in new tab page")
			: _("Edit File in new window")),
					  eap->arg, NULL, NULL, NULL, curbuf);
	    if (fname == NULL)
		goto theend;
	    eap->arg = fname;
	}
    }
    cmdmod.cmod_flags &= ~CMOD_BROWSE;	// Don't browse again in do_ecmd().
#endif

    /*
     * Either open new tab page or split the window.
     */
    if (use_tab)
    {
	if (win_new_tabpage(cmdmod.cmod_tab != 0 ? cmdmod.cmod_tab
			 : eap->addr_count == 0 ? 0
					       : (int)eap->line2 + 1) != FAIL)
	{
	    do_exedit(eap, old_curwin);

	    // set the alternate buffer for the window we came from
	    if (curwin != old_curwin
		    && win_valid(old_curwin)
		    && old_curwin->w_buffer != curbuf
		    && (cmdmod.cmod_flags & CMOD_KEEPALT) == 0)
		old_curwin->w_alt_fnum = curbuf->b_fnum;
	}
    }
    else if (win_split(eap->addr_count > 0 ? (int)eap->line2 : 0,
				     *eap->cmd == 'v' ? WSP_VERT : 0) != FAIL)
    {
	// Reset 'scrollbind' when editing another file, but keep it when
	// doing ":split" without arguments.
	if (*eap->arg != NUL)
	    RESET_BINDING(curwin);
	else
	    do_check_scrollbind(FALSE);
	do_exedit(eap, old_curwin);
    }

# ifdef FEAT_BROWSE
    cmdmod.cmod_flags = save_cmod_flags;
# endif

theend:
    vim_free(fname);
}


do_one_cmd(
    char_u	**cmdlinep,
    int		flags,
#ifdef FEAT_EVAL
    cstack_T	*cstack,
#endif
    char_u	*(*fgetline)(int, void *, int, getline_opt_T),
    void	*cookie)		// argument for fgetline()
{
    char_u	*p;
    linenr_T	lnum;
    long	n;
    char	*errormsg = NULL;	// error message
    char_u	*after_modifier = NULL;
    exarg_T	ea;			// Ex command arguments
    cmdmod_T	save_cmdmod;
    int		save_reg_executing = reg_executing;
    int		save_pending_end_reg_executing = pending_end_reg_executing;
    int		ni;			// set when Not Implemented
    char_u	*cmd;
    int		starts_with_colon = FALSE;
#ifdef FEAT_EVAL
    int		may_have_range;
    int		vim9script;
    int		did_set_expr_line = FALSE;
#endif
    int		sourcing = flags & DOCMD_VERBOSE;
    int		did_append_cmd = FALSE;

    CLEAR_FIELD(ea);
    ea.line1 = 1;
    ea.line2 = 1;
#ifdef FEAT_EVAL
    ++ex_nesting_level;
#endif

    // When the last file has not been edited :q has to be typed twice.
    if (quitmore
#ifdef FEAT_EVAL
	    // avoid that a function call in 'statusline' does this
	    && !getline_equal(fgetline, cookie, get_func_line)
#endif
	    // avoid that an autocommand, e.g. QuitPre, does this
	    && !getline_equal(fgetline, cookie, getnextac))
	--quitmore;

    /*
     * Reset browse, confirm, etc..  They are restored when returning, for
     * recursive calls.
     */
    save_cmdmod = cmdmod;

    // "#!anything" is handled like a comment.
    if ((*cmdlinep)[0] == '#' && (*cmdlinep)[1] == '!')
	goto doend;

/*
 * 1. Skip comment lines and leading white space and colons.
 * 2. Handle command modifiers.
 */
    // The "ea" structure holds the arguments that can be used.
    ea.cmd = *cmdlinep;
    ea.cmdlinep = cmdlinep;
    ea.getline = fgetline;
    ea.cookie = cookie;
#ifdef FEAT_EVAL
    ea.cstack = cstack;
    starts_with_colon = *skipwhite(ea.cmd) == ':';
#endif
    if (parse_command_modifiers(&ea, &errormsg, &cmdmod, FALSE) == FAIL)
	goto doend;
    apply_cmdmod(&cmdmod);
#ifdef FEAT_EVAL
    vim9script = in_vim9script();
#endif
    after_modifier = ea.cmd;

#ifdef FEAT_EVAL
    ea.skip = did_emsg || got_int || did_throw || (cstack->cs_idx >= 0
			 && !(cstack->cs_flags[cstack->cs_idx] & CSF_ACTIVE));
#else
    ea.skip = (if_level > 0);
#endif

/*
 * 3. Skip over the range to find the command.  Let "p" point to after it.
 *
 * We need the command to know what kind of range it uses.
 */
    cmd = ea.cmd;
#ifdef FEAT_EVAL
    // In Vim9 script a colon is required before the range.  This may also be
    // after command modifiers.
    if (vim9script && (flags & DOCMD_RANGEOK) == 0)
    {
	may_have_range = FALSE;
	for (p = ea.cmd; p >= *cmdlinep; --p)
	{
	    if (*p == ':')
		may_have_range = TRUE;
	    if (p < ea.cmd && !VIM_ISWHITE(*p))
		break;
	}
    }
    else
	may_have_range = TRUE;
    if (may_have_range)
#endif
	ea.cmd = skip_range(ea.cmd, TRUE, NULL);

#ifdef FEAT_EVAL
    if (vim9script && !may_have_range)
    {
	if (ea.cmd == cmd + 1 && *cmd == '$')
	    // should be "$VAR = val"
	    --ea.cmd;
	p = find_ex_command(&ea, NULL, lookup_scriptitem, NULL);
	if (ea.cmdidx == CMD_SIZE)
	{
	    char_u *ar = skip_range(ea.cmd, TRUE, NULL);

	    // If a ':' before the range is missing, give a clearer error
	    // message.
	    if (ar > ea.cmd && !ea.skip)
	    {
		semsg(_(e_colon_required_before_range_str), ea.cmd);
		goto doend;
	    }
	}
    }
    else
#endif
	p = find_ex_command(&ea, NULL, NULL, NULL);

#ifdef FEAT_EVAL
# ifdef FEAT_PROFILE
    // Count this line for profiling if skip is TRUE.
    if (do_profiling == PROF_YES
	    && (!ea.skip || cstack->cs_idx == 0 || (cstack->cs_idx > 0
		     && (cstack->cs_flags[cstack->cs_idx - 1] & CSF_ACTIVE))))
    {
	int skip = did_emsg || got_int || did_throw;

	if (ea.cmdidx == CMD_catch)
	    skip = !skip && !(cstack->cs_idx >= 0
			  && (cstack->cs_flags[cstack->cs_idx] & CSF_THROWN)
			  && !(cstack->cs_flags[cstack->cs_idx] & CSF_CAUGHT));
	else if (ea.cmdidx == CMD_else || ea.cmdidx == CMD_elseif)
	    skip = skip || !(cstack->cs_idx >= 0
			  && !(cstack->cs_flags[cstack->cs_idx]
						  & (CSF_ACTIVE | CSF_TRUE)));
	else if (ea.cmdidx == CMD_finally)
	    skip = FALSE;
	else if (ea.cmdidx != CMD_endif
		&& ea.cmdidx != CMD_endfor
		&& ea.cmdidx != CMD_endtry
		&& ea.cmdidx != CMD_endwhile)
	    skip = ea.skip;

	if (!skip)
	{
	    if (getline_equal(fgetline, cookie, get_func_line))
		func_line_exec(getline_cookie(fgetline, cookie));
	    else if (getline_equal(fgetline, cookie, getsourceline))
		script_line_exec();
	}
    }
# endif

    // May go to debug mode.  If this happens and the ">quit" debug command is
    // used, throw an interrupt exception and skip the next command.
    dbg_check_breakpoint(&ea);
    if (!ea.skip && got_int)
    {
	ea.skip = TRUE;
	(void)do_intthrow(cstack);
    }
#endif

/*
 * 4. parse a range specifier of the form: addr [,addr] [;addr] ..
 *
 * where 'addr' is:
 *
 * %	      (entire file)
 * $  [+-NUM]
 * 'x [+-NUM] (where x denotes a currently defined mark)
 * .  [+-NUM]
 * [+-NUM]..
 * NUM
 *
 * The ea.cmd pointer is updated to point to the first character following the
 * range spec. If an initial address is found, but no second, the upper bound
 * is equal to the lower.
 */

    // ea.addr_type for user commands is set by find_ucmd
    if (!IS_USER_CMDIDX(ea.cmdidx))
    {
	if (ea.cmdidx != CMD_SIZE)
	    ea.addr_type = cmdnames[(int)ea.cmdidx].cmd_addr_type;
	else
	    ea.addr_type = ADDR_LINES;

	// :wincmd range depends on the argument.
	if (ea.cmdidx == CMD_wincmd && p != NULL)
	    get_wincmd_addr_type(skipwhite(p), &ea);
#ifdef FEAT_QUICKFIX
	// :.cc in quickfix window uses line number
	if ((ea.cmdidx == CMD_cc || ea.cmdidx == CMD_ll) && bt_quickfix(curbuf))
	    ea.addr_type = ADDR_OTHER;
#endif
    }

    ea.cmd = cmd;
#ifdef FEAT_EVAL
    if (!may_have_range)
	ea.line1 = ea.line2 = default_address(&ea);
    else
#endif
	if (parse_cmd_address(&ea, &errormsg, FALSE) == FAIL)
	    goto doend;

/*
 * 5. Parse the command.
 */

    /*
     * Skip ':' and any white space
     */
    ea.cmd = skipwhite(ea.cmd);
    while (*ea.cmd == ':')
	ea.cmd = skipwhite(ea.cmd + 1);

    /*
     * If we got a line, but no command, then go to the line.
     * If we find a '|' or '\n' we set ea.nextcmd.
     */
    if (*ea.cmd == NUL || comment_start(ea.cmd, starts_with_colon)
			       || (ea.nextcmd = check_nextcmd(ea.cmd)) != NULL)
    {
	/*
	 * strange vi behaviour:
	 * ":3"		jumps to line 3
	 * ":3|..."	prints line 3  (not in Vim9 script)
	 * ":|"		prints current line  (not in Vim9 script)
	 */
	if (ea.skip)	    // skip this if inside :if
	    goto doend;
	errormsg = ex_range_without_command(&ea);
	goto doend;
    }

    // If this looks like an undefined user command and there are CmdUndefined
    // autocommands defined, trigger the matching autocommands.
    if (p != NULL && ea.cmdidx == CMD_SIZE && !ea.skip
	    && ASCII_ISUPPER(*ea.cmd)
	    && has_cmdundefined())
    {
	int ret;

	p = ea.cmd;
	while (ASCII_ISALNUM(*p))
	    ++p;
	p = vim_strnsave(ea.cmd, p - ea.cmd);
	ret = apply_autocmds(EVENT_CMDUNDEFINED, p, p, TRUE, NULL);
	vim_free(p);
	// If the autocommands did something and didn't cause an error, try
	// finding the command again.
	p = (ret
#ifdef FEAT_EVAL
		&& !aborting()
#endif
		) ? find_ex_command(&ea, NULL, NULL, NULL) : ea.cmd;
    }

    if (p == NULL)
    {
	if (!ea.skip)
	    errormsg = _(e_ambiguous_use_of_user_defined_command);
	goto doend;
    }
    // Check for wrong commands.
    if (*p == '!' && ea.cmd[1] == 0151 && ea.cmd[0] == 78
	    && !IS_USER_CMDIDX(ea.cmdidx))
    {
	errormsg = uc_fun_cmd();
	goto doend;
    }

    if (ea.cmdidx == CMD_SIZE)
    {
	if (!ea.skip)
	{
	    STRCPY(IObuff, _(e_not_an_editor_command));
	    if (!sourcing)
	    {
		// If the modifier was parsed OK the error must be in the
		// following command
		if (after_modifier != NULL)
		    append_command(after_modifier);
		else
		    append_command(*cmdlinep);
		did_append_cmd = TRUE;
	    }
	    errormsg = (char *)IObuff;
	    did_emsg_syntax = TRUE;
	}
	goto doend;
    }

    ni = (!IS_USER_CMDIDX(ea.cmdidx)
	    && (cmdnames[ea.cmdidx].cmd_func == ex_ni
#ifdef HAVE_EX_SCRIPT_NI
	     || cmdnames[ea.cmdidx].cmd_func == ex_script_ni
#endif
	     ));

#ifndef FEAT_EVAL
    /*
     * When the expression evaluation is disabled, recognize the ":if" and
     * ":endif" commands and ignore everything in between it.
     */
    if (ea.cmdidx == CMD_if)
	++if_level;
    if (if_level)
    {
	if (ea.cmdidx == CMD_endif)
	    --if_level;
	goto doend;
    }

#endif

    // forced commands
    if (*p == '!' && ea.cmdidx != CMD_substitute
	    && ea.cmdidx != CMD_smagic && ea.cmdidx != CMD_snomagic)
    {
	++p;
	ea.forceit = TRUE;
    }
    else
	ea.forceit = FALSE;

/*
 * 6. Parse arguments.  Then check for errors.
 */
    if (!IS_USER_CMDIDX(ea.cmdidx))
	ea.argt = (long)cmdnames[(int)ea.cmdidx].cmd_argt;

    if (!ea.skip)
    {
#ifdef HAVE_SANDBOX
	if (sandbox != 0 && !(ea.argt & EX_SBOXOK))
	{
	    // Command not allowed in sandbox.
	    errormsg = _(e_not_allowed_in_sandbox);
	    goto doend;
	}
#endif
	if (restricted != 0 && (ea.argt & EX_RESTRICT))
	{
	    errormsg = _(e_command_not_allowed_in_rvim);
	    goto doend;
	}
	if (!curbuf->b_p_ma && (ea.argt & EX_MODIFY))
	{
	    // Command not allowed in non-'modifiable' buffer
	    errormsg = _(e_cannot_make_changes_modifiable_is_off);
	    goto doend;
	}

	if (!IS_USER_CMDIDX(ea.cmdidx))
	{
#ifdef FEAT_CMDWIN
	    if (cmdwin_type != 0 && !(ea.argt & EX_CMDWIN))
	    {
		// Command not allowed in the command line window
		errormsg = _(e_invalid_in_cmdline_window);
		goto doend;
	    }
#endif
	    if (text_locked() && !(ea.argt & EX_LOCK_OK))
	    {
		// Command not allowed when text is locked
		errormsg = _(get_text_locked_msg());
		goto doend;
	    }
	}

	// Disallow editing another buffer when "curbuf_lock" is set.
	// Do allow ":checktime" (it is postponed).
	// Do allow ":edit" (check for an argument later).
	// Do allow ":file" with no arguments (check for an argument later).
	if (!(ea.argt & (EX_CMDWIN | EX_LOCK_OK))
		&& ea.cmdidx != CMD_checktime
		&& ea.cmdidx != CMD_edit
		&& ea.cmdidx != CMD_file
		&& !IS_USER_CMDIDX(ea.cmdidx)
		&& curbuf_locked())
	    goto doend;

	if (!ni && !(ea.argt & EX_RANGE) && ea.addr_count > 0)
	{
	    errormsg = _(e_no_range_allowed);
	    goto doend;
	}
    }

    if (!ni && !(ea.argt & EX_BANG) && ea.forceit)
    {
	errormsg = _(e_no_bang_allowed);
	goto doend;
    }

    /*
     * Don't complain about the range if it is not used
     * (could happen if line_count is accidentally set to 0).
     */
    if (!ea.skip && !ni && (ea.argt & EX_RANGE))
    {
	/*
	 * If the range is backwards, ask for confirmation and, if given, swap
	 * ea.line1 & ea.line2 so it's forwards again.
	 * When global command is busy, don't ask, will fail below.
	 */
	if (!global_busy && ea.line1 > ea.line2)
	{
	    if (msg_silent == 0)
	    {
		if (sourcing || exmode_active)
		{
		    errormsg = _(e_backwards_range_given);
		    goto doend;
		}
		if (ask_yesno((char_u *)
			_("Backwards range given, OK to swap"), FALSE) != 'y')
		    goto doend;
	    }
	    lnum = ea.line1;
	    ea.line1 = ea.line2;
	    ea.line2 = lnum;
	}
	if ((errormsg = invalid_range(&ea)) != NULL)
	    goto doend;
    }

    if ((ea.addr_type == ADDR_OTHER) && ea.addr_count == 0)
	// default is 1, not cursor
	ea.line2 = 1;

    correct_range(&ea);

#ifdef FEAT_FOLDING
    if (((ea.argt & EX_WHOLEFOLD) || ea.addr_count >= 2) && !global_busy
	    && ea.addr_type == ADDR_LINES)
    {
	// Put the first line at the start of a closed fold, put the last line
	// at the end of a closed fold.
	(void)hasFolding(ea.line1, &ea.line1, NULL);
	(void)hasFolding(ea.line2, NULL, &ea.line2);
    }
#endif

#ifdef FEAT_QUICKFIX
    /*
     * For the ":make" and ":grep" commands we insert the 'makeprg'/'grepprg'
     * option here, so things like % get expanded.
     */
    p = replace_makeprg(&ea, p, cmdlinep);
    if (p == NULL)
	goto doend;
#endif

    /*
     * Skip to start of argument.
     * Don't do this for the ":!" command, because ":!! -l" needs the space.
     */
    if (ea.cmdidx == CMD_bang)
	ea.arg = p;
    else
	ea.arg = skipwhite(p);

    // ":file" cannot be run with an argument when "curbuf_lock" is set
    if (ea.cmdidx == CMD_file && *ea.arg != NUL && curbuf_locked())
	goto doend;

    /*
     * Check for "++opt=val" argument.
     * Must be first, allow ":w ++enc=utf8 !cmd"
     */
    if (ea.argt & EX_ARGOPT)
	while (ea.arg[0] == '+' && ea.arg[1] == '+')
	    if (getargopt(&ea) == FAIL && !ni)
	    {
		errormsg = _(e_invalid_argument);
		goto doend;
	    }

    if (ea.cmdidx == CMD_write || ea.cmdidx == CMD_update)
    {
	if (*ea.arg == '>')			// append
	{
	    if (*++ea.arg != '>')		// typed wrong
	    {
		errormsg = _(e_use_w_or_w_gt_gt);
		goto doend;
	    }
	    ea.arg = skipwhite(ea.arg + 1);
	    ea.append = TRUE;
	}
	else if (*ea.arg == '!' && ea.cmdidx == CMD_write)  // :w !filter
	{
	    ++ea.arg;
	    ea.usefilter = TRUE;
	}
    }

    if (ea.cmdidx == CMD_read)
    {
	if (ea.forceit)
	{
	    ea.usefilter = TRUE;		// :r! filter if ea.forceit
	    ea.forceit = FALSE;
	}
	else if (*ea.arg == '!')		// :r !filter
	{
	    ++ea.arg;
	    ea.usefilter = TRUE;
	}
    }

    if (ea.cmdidx == CMD_lshift || ea.cmdidx == CMD_rshift)
    {
	ea.amount = 1;
	while (*ea.arg == *ea.cmd)		// count number of '>' or '<'
	{
	    ++ea.arg;
	    ++ea.amount;
	}
	ea.arg = skipwhite(ea.arg);
    }

    /*
     * Check for "+command" argument, before checking for next command.
     * Don't do this for ":read !cmd" and ":write !cmd".
     */
    if ((ea.argt & EX_CMDARG) && !ea.usefilter)
	ea.do_ecmd_cmd = getargcmd(&ea.arg);

    /*
     * For commands that do not use '|' inside their argument: Check for '|' to
     * separate commands and '"' or '#' to start comments.
     *
     * Otherwise: Check for <newline> to end a shell command.
     * Also do this for ":read !cmd", ":write !cmd" and ":global".
     * Also do this inside a { - } block after :command and :autocmd.
     * Any others?
     */
    if ((ea.argt & EX_TRLBAR) && !ea.usefilter)
    {
	separate_nextcmd(&ea, FALSE);
    }
    else if (ea.cmdidx == CMD_bang
	    || ea.cmdidx == CMD_terminal
	    || ea.cmdidx == CMD_global
	    || ea.cmdidx == CMD_vglobal
	    || ea.usefilter
#ifdef FEAT_EVAL
	    || inside_block(&ea)
#endif
	    )
    {
	for (p = ea.arg; *p; ++p)
	{
	    // Remove one backslash before a newline, so that it's possible to
	    // pass a newline to the shell and also a newline that is preceded
	    // with a backslash.  This makes it impossible to end a shell
	    // command in a backslash, but that doesn't appear useful.
	    // Halving the number of backslashes is incompatible with previous
	    // versions.
	    if (*p == '\\' && p[1] == '\n')
		STRMOVE(p, p + 1);
	    else if (*p == '\n' && !(ea.argt & EX_EXPR_ARG))
	    {
		ea.nextcmd = p + 1;
		*p = NUL;
		break;
	    }
	}
    }

    if ((ea.argt & EX_DFLALL) && ea.addr_count == 0)
	address_default_all(&ea);

    // accept numbered register only when no count allowed (:put)
    if (       (ea.argt & EX_REGSTR)
	    && *ea.arg != NUL
	       // Do not allow register = for user commands
	    && (!IS_USER_CMDIDX(ea.cmdidx) || *ea.arg != '=')
	    && !((ea.argt & EX_COUNT) && VIM_ISDIGIT(*ea.arg)))
    {
#ifndef FEAT_CLIPBOARD
	// check these explicitly for a more specific error message
	if (*ea.arg == '*' || *ea.arg == '+')
	{
	    errormsg = _(e_invalid_register_name);
	    goto doend;
	}
#endif
	if (valid_yank_reg(*ea.arg, (ea.cmdidx != CMD_put
					      && !IS_USER_CMDIDX(ea.cmdidx))))
	{
	    ea.regname = *ea.arg++;
#ifdef FEAT_EVAL
	    // for '=' register: accept the rest of the line as an expression
	    if (ea.arg[-1] == '=' && ea.arg[0] != NUL)
	    {
		if (!ea.skip)
		{
		    set_expr_line(vim_strsave(ea.arg), &ea);
		    did_set_expr_line = TRUE;
		}
		ea.arg += STRLEN(ea.arg);
	    }
#endif
	    ea.arg = skipwhite(ea.arg);
	}
    }

    /*
     * Check for a count.  When accepting a EX_BUFNAME, don't use "123foo" as a
     * count, it's a buffer name.
     */
    if ((ea.argt & EX_COUNT) && VIM_ISDIGIT(*ea.arg)
	    && (!(ea.argt & EX_BUFNAME) || *(p = skipdigits(ea.arg + 1)) == NUL
							  || VIM_ISWHITE(*p)))
    {
	n = getdigits_quoted(&ea.arg);
	ea.arg = skipwhite(ea.arg);
	if (n <= 0 && !ni && (ea.argt & EX_ZEROR) == 0)
	{
	    errormsg = _(e_positive_count_required);
	    goto doend;
	}
	if (ea.addr_type != ADDR_LINES)	// e.g. :buffer 2, :sleep 3
	{
	    ea.line2 = n;
	    if (ea.addr_count == 0)
		ea.addr_count = 1;
	}
	else
	{
	    ea.line1 = ea.line2;
	    if (ea.line2 >= LONG_MAX - (n - 1))
		ea.line2 = LONG_MAX;  // avoid overflow
	    else
		ea.line2 += n - 1;
	    ++ea.addr_count;
	    /*
	     * Be vi compatible: no error message for out of range.
	     */
	    if (ea.line2 > curbuf->b_ml.ml_line_count)
		ea.line2 = curbuf->b_ml.ml_line_count;
	}
    }

    /*
     * Check for flags: 'l', 'p' and '#'.
     */
    if (ea.argt & EX_FLAGS)
	get_flags(&ea);
    if (!ni && !(ea.argt & EX_EXTRA) && *ea.arg != NUL
	    && *ea.arg != '"' && (*ea.arg != '|' || (ea.argt & EX_TRLBAR) == 0))
    {
	// no arguments allowed but there is something
	errormsg = ex_errmsg(e_trailing_characters_str, ea.arg);
	goto doend;
    }

    if (!ni && (ea.argt & EX_NEEDARG) && *ea.arg == NUL)
    {
	errormsg = _(e_argument_required);
	goto doend;
    }

#ifdef FEAT_EVAL
    /*
     * Skip the command when it's not going to be executed.
     * The commands like :if, :endif, etc. always need to be executed.
     * Also make an exception for commands that handle a trailing command
     * themselves.
     */
    if (ea.skip)
    {
	switch (ea.cmdidx)
	{
	    // commands that need evaluation
	    case CMD_while:
	    case CMD_endwhile:
	    case CMD_for:
	    case CMD_endfor:
	    case CMD_if:
	    case CMD_elseif:
	    case CMD_else:
	    case CMD_endif:
	    case CMD_try:
	    case CMD_catch:
	    case CMD_finally:
	    case CMD_endtry:
	    case CMD_function:
	    case CMD_def:
				break;

	    // Commands that handle '|' themselves.  Check: A command should
	    // either have the EX_TRLBAR flag, appear in this list or appear in
	    // the list at ":help :bar".
	    case CMD_aboveleft:
	    case CMD_and:
	    case CMD_belowright:
	    case CMD_botright:
	    case CMD_browse:
	    case CMD_call:
	    case CMD_confirm:
	    case CMD_const:
	    case CMD_delfunction:
	    case CMD_djump:
	    case CMD_dlist:
	    case CMD_dsearch:
	    case CMD_dsplit:
	    case CMD_echo:
	    case CMD_echoerr:
	    case CMD_echomsg:
	    case CMD_echon:
	    case CMD_eval:
	    case CMD_execute:
	    case CMD_filter:
	    case CMD_final:
	    case CMD_help:
	    case CMD_hide:
	    case CMD_horizontal:
	    case CMD_ijump:
	    case CMD_ilist:
	    case CMD_isearch:
	    case CMD_isplit:
	    case CMD_keepalt:
	    case CMD_keepjumps:
	    case CMD_keepmarks:
	    case CMD_keeppatterns:
	    case CMD_leftabove:
	    case CMD_let:
	    case CMD_lockmarks:
	    case CMD_lockvar:
	    case CMD_lua:
	    case CMD_match:
	    case CMD_mzscheme:
	    case CMD_noautocmd:
	    case CMD_noswapfile:
	    case CMD_perl:
	    case CMD_psearch:
	    case CMD_py3:
	    case CMD_python3:
	    case CMD_python:
	    case CMD_return:
	    case CMD_rightbelow:
	    case CMD_ruby:
	    case CMD_silent:
	    case CMD_smagic:
	    case CMD_snomagic:
	    case CMD_substitute:
	    case CMD_syntax:
	    case CMD_tab:
	    case CMD_tcl:
	    case CMD_throw:
	    case CMD_tilde:
	    case CMD_topleft:
	    case CMD_unlet:
	    case CMD_unlockvar:
	    case CMD_var:
	    case CMD_verbose:
	    case CMD_vertical:
	    case CMD_wincmd:
				break;

	    default:		goto doend;
	}
    }
#endif

    if (ea.argt & EX_XFILE)
    {
	if (expand_filename(&ea, cmdlinep, &errormsg) == FAIL)
	    goto doend;
    }

    /*
     * Accept buffer name.  Cannot be used at the same time with a buffer
     * number.  Don't do this for a user command.
     */
    if ((ea.argt & EX_BUFNAME) && *ea.arg != NUL && ea.addr_count == 0
	    && !IS_USER_CMDIDX(ea.cmdidx))
    {
	/*
	 * :bdelete, :bwipeout and :bunload take several arguments, separated
	 * by spaces: find next space (skipping over escaped characters).
	 * The others take one argument: ignore trailing spaces.
	 */
	if (ea.cmdidx == CMD_bdelete || ea.cmdidx == CMD_bwipeout
						  || ea.cmdidx == CMD_bunload)
	    p = skiptowhite_esc(ea.arg);
	else
	{
	    p = ea.arg + STRLEN(ea.arg);
	    while (p > ea.arg && VIM_ISWHITE(p[-1]))
		--p;
	}
	ea.line2 = buflist_findpat(ea.arg, p, (ea.argt & EX_BUFUNL) != 0,
								FALSE, FALSE);
	if (ea.line2 < 0)	    // failed
	    goto doend;
	ea.addr_count = 1;
	ea.arg = skipwhite(p);
    }

    // The :try command saves the emsg_silent flag, reset it here when
    // ":silent! try" was used, it should only apply to :try itself.
    if (ea.cmdidx == CMD_try && cmdmod.cmod_did_esilent > 0)
    {
	emsg_silent -= cmdmod.cmod_did_esilent;
	if (emsg_silent < 0)
	    emsg_silent = 0;
	cmdmod.cmod_did_esilent = 0;
    }

/*
 * 7. Execute the command.
 */

    if (IS_USER_CMDIDX(ea.cmdidx))
    {
	/*
	 * Execute a user-defined command.
	 */
	do_ucmd(&ea);
    }
    else
    {
	/*
	 * Call the function to execute the builtin command.
	 */
	ea.errmsg = NULL;
	(cmdnames[ea.cmdidx].cmd_func)(&ea);
	if (ea.errmsg != NULL)
	    errormsg = ea.errmsg;
    }

#ifdef FEAT_EVAL
    // Set flag that any command was executed, used by ex_vim9script().
    // Not if this was a command that wasn't executed or :endif.
    if (sourcing_a_script(&ea)
	    && current_sctx.sc_sid > 0
	    && ea.cmdidx != CMD_endif
	    && (cstack->cs_idx < 0
		    || (cstack->cs_flags[cstack->cs_idx] & CSF_ACTIVE)))
	SCRIPT_ITEM(current_sctx.sc_sid)->sn_state = SN_STATE_HAD_COMMAND;

    /*
     * If the command just executed called do_cmdline(), any throw or ":return"
     * or ":finish" encountered there must also check the cstack of the still
     * active do_cmdline() that called this do_one_cmd().  Rethrow an uncaught
     * exception, or reanimate a returned function or finished script file and
     * return or finish it again.
     */
    if (need_rethrow)
	do_throw(cstack);
    else if (check_cstack)
    {
	if (source_finished(fgetline, cookie))
	    do_finish(&ea, TRUE);
	else if (getline_equal(fgetline, cookie, get_func_line)
						   && current_func_returned())
	    do_return(&ea, TRUE, FALSE, NULL);
    }
    need_rethrow = check_cstack = FALSE;
#endif

doend:
    if (curwin->w_cursor.lnum == 0)	// can happen with zero line number
    {
	curwin->w_cursor.lnum = 1;
	curwin->w_cursor.col = 0;
    }

    if (errormsg != NULL && *errormsg != NUL && !did_emsg)
    {
	if ((sourcing || !KeyTyped) && !did_append_cmd)
	{
	    if (errormsg != (char *)IObuff)
	    {
		STRCPY(IObuff, errormsg);
		errormsg = (char *)IObuff;
	    }
	    append_command(*cmdlinep);
	}
	emsg(errormsg);
    }
#ifdef FEAT_EVAL
    do_errthrow(cstack,
	    (ea.cmdidx != CMD_SIZE && !IS_USER_CMDIDX(ea.cmdidx))
			? cmdnames[(int)ea.cmdidx].cmd_name : (char_u *)NULL);

    if (did_set_expr_line)
	set_expr_line(NULL, NULL);
#endif

    undo_cmdmod(&cmdmod);
    cmdmod = save_cmdmod;
    reg_executing = save_reg_executing;
    pending_end_reg_executing = save_pending_end_reg_executing;

    if (ea.nextcmd && *ea.nextcmd == NUL)	// not really a next command
	ea.nextcmd = NULL;

#ifdef FEAT_EVAL
    --ex_nesting_level;
    vim_free(ea.cmdline_tofree);
#endif

    return ea.nextcmd;
}


pxa3xx_gcu_handle_irq(int irq, void *ctx)
{
	struct pxa3xx_gcu_priv *priv = ctx;
	struct pxa3xx_gcu_shared *shared = priv->shared;
	u32 status = gc_readl(priv, REG_GCISCR) & IE_ALL;

	QDUMP("-Interrupt");

	if (!status)
		return IRQ_NONE;

	spin_lock(&priv->spinlock);
	shared->num_interrupts++;

	if (status & IE_EEOB) {
		QDUMP(" [EEOB]");

		flush_running(priv);
		wake_up_all(&priv->wait_free);

		if (priv->ready) {
			run_ready(priv);
		} else {
			/* There is no more data prepared by the userspace.
			 * Set hw_running = 0 and wait for the next userspace
			 * kick-off */
			shared->num_idle++;
			shared->hw_running = 0;

			QDUMP(" '-> Idle.");

			/* set ring buffer length to zero */
			gc_writel(priv, REG_GCRBLR, 0);

			wake_up_all(&priv->wait_idle);
		}

		shared->num_done++;
	} else {
		QERROR(" [???]");
		dump_whole_state(priv);
	}

	/* Clear the interrupt */
	gc_writel(priv, REG_GCISCR, status);
	spin_unlock(&priv->spinlock);

	return IRQ_HANDLED;
}


static int pxa3xx_gcu_open(struct inode *inode, struct file *file)
{
	return 0;
}


JBIG2Bitmap *JBIG2Bitmap::getSlice(unsigned int x, unsigned int y, unsigned int wA, unsigned int hA)
{
    JBIG2Bitmap *slice;
    unsigned int xx, yy;

    if (!data) {
        return nullptr;
    }

    slice = new JBIG2Bitmap(0, wA, hA);
    if (slice->isOk()) {
        slice->clearToZero();
        for (yy = 0; yy < hA; ++yy) {
            for (xx = 0; xx < wA; ++xx) {
                if (getPixel(x + xx, y + yy)) {
                    slice->setPixel(xx, yy);
                }
            }
        }
    } else {
        delete slice;
        slice = nullptr;
    }
    return slice;
}


void JBIG2Stream::readExtensionSeg(unsigned int length)
{
    // skip the segment
    byteCounter += curStr->discardChars(length);
}


int JBIG2Stream::getChars(int nChars, unsigned char *buffer)
{
    int n, i;

    if (nChars <= 0 || !dataPtr) {
        return 0;
    }
    if (dataEnd - dataPtr < nChars) {
        n = (int)(dataEnd - dataPtr);
    } else {
        n = nChars;
    }
    for (i = 0; i < n; ++i) {
        buffer[i] = *dataPtr++ ^ 0xff;
    }
    return n;
}


void JBIG2Bitmap::combine(JBIG2Bitmap *bitmap, int x, int y, unsigned int combOp)
{
    int x0, x1, y0, y1, xx, yy;
    unsigned char *srcPtr, *destPtr;
    unsigned int src0, src1, src, dest, s1, s2, m1, m2, m3;
    bool oneByte;

    // check for the pathological case where y = -2^31
    if (y < -0x7fffffff) {
        return;
    }
    if (y < 0) {
        y0 = -y;
    } else {
        y0 = 0;
    }
    if (y + bitmap->h > h) {
        y1 = h - y;
    } else {
        y1 = bitmap->h;
    }
    if (y0 >= y1) {
        return;
    }

    if (x >= 0) {
        x0 = x & ~7;
    } else {
        x0 = 0;
    }
    if (unlikely(checkedAdd(x, bitmap->w, &x1))) {
        return;
    }
    if (x1 > w) {
        x1 = w;
    }
    if (x0 >= x1) {
        return;
    }

    s1 = x & 7;
    s2 = 8 - s1;
    m1 = 0xff >> (x1 & 7);
    m2 = 0xff << (((x1 & 7) == 0) ? 0 : 8 - (x1 & 7));
    m3 = (0xff >> s1) & m2;

    oneByte = x0 == ((x1 - 1) & ~7);

    for (yy = y0; yy < y1; ++yy) {
        if (unlikely((y + yy >= h) || (y + yy < 0))) {
            continue;
        }

        // one byte per line -- need to mask both left and right side
        if (oneByte) {
            if (x >= 0) {
                destPtr = data + (y + yy) * line + (x >> 3);
                srcPtr = bitmap->data + yy * bitmap->line;
                dest = *destPtr;
                src1 = *srcPtr;
                switch (combOp) {
                case 0: // or
                    dest |= (src1 >> s1) & m2;
                    break;
                case 1: // and
                    dest &= ((0xff00 | src1) >> s1) | m1;
                    break;
                case 2: // xor
                    dest ^= (src1 >> s1) & m2;
                    break;
                case 3: // xnor
                    dest ^= ((src1 ^ 0xff) >> s1) & m2;
                    break;
                case 4: // replace
                    dest = (dest & ~m3) | ((src1 >> s1) & m3);
                    break;
                }
                *destPtr = dest;
            } else {
                destPtr = data + (y + yy) * line;
                srcPtr = bitmap->data + yy * bitmap->line + (-x >> 3);
                dest = *destPtr;
                src1 = *srcPtr;
                switch (combOp) {
                case 0: // or
                    dest |= src1 & m2;
                    break;
                case 1: // and
                    dest &= src1 | m1;
                    break;
                case 2: // xor
                    dest ^= src1 & m2;
                    break;
                case 3: // xnor
                    dest ^= (src1 ^ 0xff) & m2;
                    break;
                case 4: // replace
                    dest = (src1 & m2) | (dest & m1);
                    break;
                }
                *destPtr = dest;
            }

            // multiple bytes per line -- need to mask left side of left-most
            // byte and right side of right-most byte
        } else {

            // left-most byte
            if (x >= 0) {
                destPtr = data + (y + yy) * line + (x >> 3);
                srcPtr = bitmap->data + yy * bitmap->line;
                src1 = *srcPtr++;
                dest = *destPtr;
                switch (combOp) {
                case 0: // or
                    dest |= src1 >> s1;
                    break;
                case 1: // and
                    dest &= (0xff00 | src1) >> s1;
                    break;
                case 2: // xor
                    dest ^= src1 >> s1;
                    break;
                case 3: // xnor
                    dest ^= (src1 ^ 0xff) >> s1;
                    break;
                case 4: // replace
                    dest = (dest & (0xff << s2)) | (src1 >> s1);
                    break;
                }
                *destPtr++ = dest;
                xx = x0 + 8;
            } else {
                destPtr = data + (y + yy) * line;
                srcPtr = bitmap->data + yy * bitmap->line + (-x >> 3);
                src1 = *srcPtr++;
                xx = x0;
            }

            // middle bytes
            for (; xx < x1 - 8; xx += 8) {
                dest = *destPtr;
                src0 = src1;
                src1 = *srcPtr++;
                src = (((src0 << 8) | src1) >> s1) & 0xff;
                switch (combOp) {
                case 0: // or
                    dest |= src;
                    break;
                case 1: // and
                    dest &= src;
                    break;
                case 2: // xor
                    dest ^= src;
                    break;
                case 3: // xnor
                    dest ^= src ^ 0xff;
                    break;
                case 4: // replace
                    dest = src;
                    break;
                }
                *destPtr++ = dest;
            }

            // right-most byte
            // note: this last byte (src1) may not actually be used, depending
            // on the values of s1, m1, and m2 - and in fact, it may be off
            // the edge of the source bitmap, which means we need to allocate
            // one extra guard byte at the end of each bitmap
            dest = *destPtr;
            src0 = src1;
            src1 = *srcPtr++;
            src = (((src0 << 8) | src1) >> s1) & 0xff;
            switch (combOp) {
            case 0: // or
                dest |= src & m2;
                break;
            case 1: // and
                dest &= src | m1;
                break;
            case 2: // xor
                dest ^= src & m2;
                break;
            case 3: // xnor
                dest ^= (src ^ 0xff) & m2;
                break;
            case 4: // replace
                dest = (src & m2) | (dest & m1);
                break;
            }
            *destPtr = dest;
        }
    }
}


int JBIG2MMRDecoder::getWhiteCode()
{
    const CCITTCode *p;
    unsigned int code;

    if (bufLen == 0) {
        buf = str->getChar() & 0xff;
        bufLen = 8;
        ++nBytesRead;
        ++byteCounter;
    }
    while (true) {
        if (bufLen >= 11 && ((buf >> (bufLen - 7)) & 0x7f) == 0) {
            if (bufLen <= 12) {
                code = buf << (12 - bufLen);
            } else {
                code = buf >> (bufLen - 12);
            }
            p = &whiteTab1[code & 0x1f];
        } else {
            if (bufLen <= 9) {
                code = buf << (9 - bufLen);
            } else {
                code = buf >> (bufLen - 9);
            }
            p = &whiteTab2[code & 0x1ff];
        }
        if (p->bits > 0 && p->bits <= (int)bufLen) {
            bufLen -= p->bits;
            return p->n;
        }
        if (bufLen >= 12) {
            break;
        }
        buf = (buf << 8) | (str->getChar() & 0xff);
        bufLen += 8;
        ++nBytesRead;
        ++byteCounter;
    }
    error(errSyntaxError, str->getPos(), "Bad white code in JBIG2 MMR stream");
    // eat a bit and return a positive number so that the caller doesn't
    // go into an infinite loop
    --bufLen;
    return 1;
}


void JBIG2Stream::readHalftoneRegionSeg(unsigned int segNum, bool imm, bool lossless, unsigned int length, unsigned int *refSegs, unsigned int nRefSegs)
{
    std::unique_ptr<JBIG2Bitmap> bitmap;
    JBIG2Segment *seg;
    JBIG2PatternDict *patternDict;
    std::unique_ptr<JBIG2Bitmap> skipBitmap;
    unsigned int *grayImg;
    JBIG2Bitmap *patternBitmap;
    unsigned int w, h, x, y, segInfoFlags, extCombOp;
    unsigned int flags, mmr, templ, enableSkip, combOp;
    unsigned int gridW, gridH, stepX, stepY, patW, patH;
    int atx[4], aty[4];
    int gridX, gridY, xx, yy, bit, j;
    unsigned int bpp, m, n, i;

    // region segment info field
    if (!readULong(&w) || !readULong(&h) || !readULong(&x) || !readULong(&y) || !readUByte(&segInfoFlags)) {
        goto eofError;
    }
    extCombOp = segInfoFlags & 7;

    // rest of the halftone region header
    if (!readUByte(&flags)) {
        goto eofError;
    }
    mmr = flags & 1;
    templ = (flags >> 1) & 3;
    enableSkip = (flags >> 3) & 1;
    combOp = (flags >> 4) & 7;
    if (!readULong(&gridW) || !readULong(&gridH) || !readLong(&gridX) || !readLong(&gridY) || !readUWord(&stepX) || !readUWord(&stepY)) {
        goto eofError;
    }
    if (w == 0 || h == 0 || w >= INT_MAX / h) {
        error(errSyntaxError, curStr->getPos(), "Bad bitmap size in JBIG2 halftone segment");
        return;
    }
    if (gridH == 0 || gridW >= INT_MAX / gridH) {
        error(errSyntaxError, curStr->getPos(), "Bad grid size in JBIG2 halftone segment");
        return;
    }

    // get pattern dictionary
    if (nRefSegs != 1) {
        error(errSyntaxError, curStr->getPos(), "Bad symbol dictionary reference in JBIG2 halftone segment");
        return;
    }
    seg = findSegment(refSegs[0]);
    if (seg == nullptr || seg->getType() != jbig2SegPatternDict) {
        error(errSyntaxError, curStr->getPos(), "Bad symbol dictionary reference in JBIG2 halftone segment");
        return;
    }

    patternDict = (JBIG2PatternDict *)seg;
    i = patternDict->getSize();
    if (i <= 1) {
        bpp = 0;
    } else {
        --i;
        bpp = 0;
        // i = floor((size-1) / 2^bpp)
        while (i > 0) {
            ++bpp;
            i >>= 1;
        }
    }
    patternBitmap = patternDict->getBitmap(0);
    if (unlikely(patternBitmap == nullptr)) {
        error(errSyntaxError, curStr->getPos(), "Bad pattern bitmap");
        return;
    }
    patW = patternBitmap->getWidth();
    patH = patternBitmap->getHeight();

    // set up the arithmetic decoder
    if (!mmr) {
        resetGenericStats(templ, nullptr);
        arithDecoder->start();
    }

    // allocate the bitmap
    bitmap = std::make_unique<JBIG2Bitmap>(segNum, w, h);
    if (flags & 0x80) { // HDEFPIXEL
        bitmap->clearToOne();
    } else {
        bitmap->clearToZero();
    }

    // compute the skip bitmap
    if (enableSkip) {
        skipBitmap = std::make_unique<JBIG2Bitmap>(0, gridW, gridH);
        skipBitmap->clearToZero();
        for (m = 0; m < gridH; ++m) {
            for (n = 0; n < gridW; ++n) {
                xx = gridX + m * stepY + n * stepX;
                yy = gridY + m * stepX - n * stepY;
                if (((xx + (int)patW) >> 8) <= 0 || (xx >> 8) >= (int)w || ((yy + (int)patH) >> 8) <= 0 || (yy >> 8) >= (int)h) {
                    skipBitmap->setPixel(n, m);
                }
            }
        }
    }

    // read the gray-scale image
    grayImg = (unsigned int *)gmallocn_checkoverflow(gridW * gridH, sizeof(unsigned int));
    if (!grayImg) {
        return;
    }
    memset(grayImg, 0, gridW * gridH * sizeof(unsigned int));
    atx[0] = templ <= 1 ? 3 : 2;
    aty[0] = -1;
    atx[1] = -3;
    aty[1] = -1;
    atx[2] = 2;
    aty[2] = -2;
    atx[3] = -2;
    aty[3] = -2;
    for (j = bpp - 1; j >= 0; --j) {
        std::unique_ptr<JBIG2Bitmap> grayBitmap = readGenericBitmap(mmr, gridW, gridH, templ, false, enableSkip, skipBitmap.get(), atx, aty, -1);
        i = 0;
        for (m = 0; m < gridH; ++m) {
            for (n = 0; n < gridW; ++n) {
                bit = grayBitmap->getPixel(n, m) ^ (grayImg[i] & 1);
                grayImg[i] = (grayImg[i] << 1) | bit;
                ++i;
            }
        }
    }

    // decode the image
    i = 0;
    for (m = 0; m < gridH; ++m) {
        xx = gridX + m * stepY;
        yy = gridY + m * stepX;
        for (n = 0; n < gridW; ++n) {
            if (!(enableSkip && skipBitmap->getPixel(n, m))) {
                patternBitmap = patternDict->getBitmap(grayImg[i]);
                if (unlikely(patternBitmap == nullptr)) {
                    gfree(grayImg);
                    error(errSyntaxError, curStr->getPos(), "Bad pattern bitmap");
                    return;
                }
                bitmap->combine(patternBitmap, xx >> 8, yy >> 8, combOp);
            }
            xx += stepX;
            yy -= stepY;
            ++i;
        }
    }

    gfree(grayImg);

    // combine the region bitmap into the page bitmap
    if (imm) {
        if (pageH == 0xffffffff && y + h > curPageH) {
            pageBitmap->expand(y + h, pageDefPixel);
        }
        pageBitmap->combine(bitmap.get(), x, y, extCombOp);

        // store the region bitmap
    } else {
        segments.push_back(std::move(bitmap));
    }

    return;

eofError:
    error(errSyntaxError, curStr->getPos(), "Unexpected EOF in JBIG2 stream");
}


JBIG2SegmentType getType() override { return jbig2SegPatternDict; }


void JBIG2Bitmap::expand(int newH, unsigned int pixel)
{
    if (unlikely(!data)) {
        return;
    }
    if (newH <= h || line <= 0 || newH >= (INT_MAX - 1) / line) {
        error(errSyntaxError, -1, "invalid width/height");
        gfree(data);
        data = nullptr;
        return;
    }
    // need to allocate one extra guard byte for use in combine()
    data = (unsigned char *)grealloc(data, newH * line + 1);
    if (pixel) {
        memset(data + h * line, 0xff, (newH - h) * line);
    } else {
        memset(data + h * line, 0x00, (newH - h) * line);
    }
    h = newH;
    data[h * line] = 0;
}


JBIG2MMRDecoder::JBIG2MMRDecoder()
{
    str = nullptr;
    byteCounter = 0;
    reset();
}


JBIG2HuffmanTable *getHuffTable() { return table; }


JBIG2Bitmap::~JBIG2Bitmap()
{
    gfree(data);
}


static HtPU *create_path_to_index(RzBuffer *cache_buf, cache_img_t *img, cache_hdr_t *hdr) {
	HtPU *path_to_idx = ht_pu_new0();
	if (!path_to_idx) {
		return NULL;
	}
	for (size_t i = 0; i != hdr->imagesCount; i++) {
		char file[256];
		if (rz_buf_read_at(cache_buf, img[i].pathFileOffset, (ut8 *)&file, sizeof(file)) != sizeof(file)) {
			continue;
		}
		file[255] = 0;
		ht_pu_insert(path_to_idx, file, (ut64)i);
	}

	return path_to_idx;
}


static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu)
{
	u64 debugctl = vmx_supported_debugctl();

	if (!intel_pmu_lbr_is_enabled(vcpu))
		debugctl &= ~DEBUGCTLMSR_LBR_MASK;

	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
		debugctl &= ~DEBUGCTLMSR_BUS_LOCK_DETECT;

	return debugctl;
}


static void vmx_exit(void)
{
#ifdef CONFIG_KEXEC_CORE
	RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
	synchronize_rcu();
#endif

	kvm_exit();

#if IS_ENABLED(CONFIG_HYPERV)
	if (static_branch_unlikely(&enable_evmcs)) {
		int cpu;
		struct hv_vp_assist_page *vp_ap;
		/*
		 * Reset everything to support using non-enlightened VMCS
		 * access later (e.g. when we reload the module with
		 * enlightened_vmcs=0)
		 */
		for_each_online_cpu(cpu) {
			vp_ap =	hv_get_vp_assist_page(cpu);

			if (!vp_ap)
				continue;

			vp_ap->nested_control.features.directhypercall = 0;
			vp_ap->current_nested_vmcs = 0;
			vp_ap->enlighten_vmentry = 0;
		}

		static_branch_disable(&enable_evmcs);
	}
#endif
	vmx_cleanup_l1d_flush();

	allow_smaller_maxphyaddr = false;
}


static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
{
	struct vcpu_vmx *vmx = to_vmx(vcpu);

	if (is_guest_mode(vcpu)) {
		vmx->nested.update_vmcs01_apicv_status = true;
		return;
	}

	pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
	if (cpu_has_secondary_exec_ctrls()) {
		if (kvm_vcpu_apicv_active(vcpu))
			secondary_exec_controls_setbit(vmx,
				      SECONDARY_EXEC_APIC_REGISTER_VIRT |
				      SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
		else
			secondary_exec_controls_clearbit(vmx,
					SECONDARY_EXEC_APIC_REGISTER_VIRT |
					SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
	}

	vmx_update_msr_bitmap_x2apic(vcpu);
}


static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
{
	u8 cache;

	/* We wanted to honor guest CD/MTRR/PAT, but doing so could result in
	 * memory aliases with conflicting memory types and sometimes MCEs.
	 * We have to be careful as to what are honored and when.
	 *
	 * For MMIO, guest CD/MTRR are ignored.  The EPT memory type is set to
	 * UC.  The effective memory type is UC or WC depending on guest PAT.
	 * This was historically the source of MCEs and we want to be
	 * conservative.
	 *
	 * When there is no need to deal with noncoherent DMA (e.g., no VT-d
	 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored.  The
	 * EPT memory type is set to WB.  The effective memory type is forced
	 * WB.
	 *
	 * Otherwise, we trust guest.  Guest CD/MTRR/PAT are all honored.  The
	 * EPT memory type is used to emulate guest CD/MTRR.
	 */

	if (is_mmio)
		return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;

	if (!kvm_arch_has_noncoherent_dma(vcpu->kvm))
		return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT;

	if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
		if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
			cache = MTRR_TYPE_WRBACK;
		else
			cache = MTRR_TYPE_UNCACHABLE;

		return (cache << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT;
	}

	return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT;
}


static void init_vmcs(struct vcpu_vmx *vmx)
{
	if (nested)
		nested_vmx_set_vmcs_shadowing_bitmap();

	if (cpu_has_vmx_msr_bitmap())
		vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));

	vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */

	/* Control */
	pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));

	exec_controls_set(vmx, vmx_exec_control(vmx));

	if (cpu_has_secondary_exec_ctrls())
		secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx));

	if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) {
		vmcs_write64(EOI_EXIT_BITMAP0, 0);
		vmcs_write64(EOI_EXIT_BITMAP1, 0);
		vmcs_write64(EOI_EXIT_BITMAP2, 0);
		vmcs_write64(EOI_EXIT_BITMAP3, 0);

		vmcs_write16(GUEST_INTR_STATUS, 0);

		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
		vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
	}

	if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
		vmcs_write32(PLE_GAP, ple_gap);
		vmx->ple_window = ple_window;
		vmx->ple_window_dirty = true;
	}

	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
	vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */

	vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
	vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
	vmx_set_constant_host_state(vmx);
	vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
	vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */

	if (cpu_has_vmx_vmfunc())
		vmcs_write64(VM_FUNCTION_CONTROL, 0);

	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));

	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);

	vm_exit_controls_set(vmx, vmx_vmexit_ctrl());

	/* 22.2.1, 20.8.1 */
	vm_entry_controls_set(vmx, vmx_vmentry_ctrl());

	vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS;
	vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits);

	set_cr4_guest_host_mask(vmx);

	if (vmx->vpid != 0)
		vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);

	if (cpu_has_vmx_xsaves())
		vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);

	if (enable_pml) {
		vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
		vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
	}

	vmx_write_encls_bitmap(&vmx->vcpu, NULL);

	if (vmx_pt_mode_is_host_guest()) {
		memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
		/* Bit[6~0] are forced to 1, writes are ignored. */
		vmx->pt_desc.guest.output_mask = 0x7F;
		vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
	}

	vmcs_write32(GUEST_SYSENTER_CS, 0);
	vmcs_writel(GUEST_SYSENTER_ESP, 0);
	vmcs_writel(GUEST_SYSENTER_EIP, 0);
	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);

	if (cpu_has_vmx_tpr_shadow()) {
		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
		if (cpu_need_tpr_shadow(&vmx->vcpu))
			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
				     __pa(vmx->vcpu.arch.apic->regs));
		vmcs_write32(TPR_THRESHOLD, 0);
	}

	vmx_setup_uret_msrs(vmx);
}


static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
{
	struct vcpu_svm *svm = to_svm(vcpu);

	/*
	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
	 * get that intercept, this function will be called again though and
	 * we'll get the vintr intercept. However, if the vGIF feature is
	 * enabled, the STGI interception will not occur. Enable the irq
	 * window under the assumption that the hardware will set the GIF.
	 */
	if (vgif || gif_set(svm)) {
		/*
		 * IRQ window is not needed when AVIC is enabled,
		 * unless we have pending ExtINT since it cannot be injected
		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
		 * and fallback to injecting IRQ via V_IRQ.
		 *
		 * If running nested, AVIC is already locally inhibited
		 * on this vCPU, therefore there is no need to request
		 * the VM wide AVIC inhibition.
		 */
		if (!is_guest_mode(vcpu))
			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);

		svm_set_vintr(svm);
	}
}


static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl)
{
	/*
	 * These bits in the secondary execution controls field
	 * are dynamic, the others are mostly based on the hypervisor
	 * architecture and the guest's CPUID.  Do not touch the
	 * dynamic bits.
	 */
	u32 mask =
		SECONDARY_EXEC_SHADOW_VMCS |
		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
		SECONDARY_EXEC_DESC;

	u32 cur_ctl = secondary_exec_controls_get(vmx);

	secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask));
}