MagickExport Image *AcquireImage(const ImageInfo *image_info,
  ExceptionInfo *exception)
{
  const char
    *option;

  Image
    *image;

  MagickStatusType
    flags;

  /*
    Allocate image structure.
  */
  (void) LogMagickEvent(TraceEvent,GetMagickModule(),"...");
  image=(Image *) AcquireCriticalMemory(sizeof(*image));
  (void) memset(image,0,sizeof(*image));
  /*
    Initialize Image structure.
  */
  (void) CopyMagickString(image->magick,"MIFF",MagickPathExtent);
  image->storage_class=DirectClass;
  image->depth=MAGICKCORE_QUANTUM_DEPTH;
  image->colorspace=sRGBColorspace;
  image->rendering_intent=PerceptualIntent;
  image->gamma=1.000f/2.200f;
  image->chromaticity.red_primary.x=0.6400f;
  image->chromaticity.red_primary.y=0.3300f;
  image->chromaticity.red_primary.z=0.0300f;
  image->chromaticity.green_primary.x=0.3000f;
  image->chromaticity.green_primary.y=0.6000f;
  image->chromaticity.green_primary.z=0.1000f;
  image->chromaticity.blue_primary.x=0.1500f;
  image->chromaticity.blue_primary.y=0.0600f;
  image->chromaticity.blue_primary.z=0.7900f;
  image->chromaticity.white_point.x=0.3127f;
  image->chromaticity.white_point.y=0.3290f;
  image->chromaticity.white_point.z=0.3583f;
  image->interlace=NoInterlace;
  image->ticks_per_second=UndefinedTicksPerSecond;
  image->compose=OverCompositeOp;
  (void) QueryColorCompliance(MatteColor,AllCompliance,&image->matte_color,
    exception);
  (void) QueryColorCompliance(BackgroundColor,AllCompliance,
    &image->background_color,exception);
  (void) QueryColorCompliance(BorderColor,AllCompliance,&image->border_color,
    exception);
  (void) QueryColorCompliance(TransparentColor,AllCompliance,
    &image->transparent_color,exception);
  GetTimerInfo(&image->timer);
  image->cache=AcquirePixelCache(0);
  image->channel_mask=DefaultChannels;
  image->channel_map=AcquirePixelChannelMap();
  image->blob=CloneBlobInfo((BlobInfo *) NULL);
  image->timestamp=GetMagickTime();
  image->debug=IsEventLogging();
  image->reference_count=1;
  image->semaphore=AcquireSemaphoreInfo();
  image->signature=MagickCoreSignature;
  if (image_info == (ImageInfo *) NULL)
    return(image);
  /*
    Transfer image info.
  */
  SetBlobExempt(image,image_info->file != (FILE *) NULL ? MagickTrue :
    MagickFalse);
  (void) CopyMagickString(image->filename,image_info->filename,
    MagickPathExtent);
  (void) CopyMagickString(image->magick_filename,image_info->filename,
    MagickPathExtent);
  (void) CopyMagickString(image->magick,image_info->magick,MagickPathExtent);
  if (image_info->size != (char *) NULL)
    {
      (void) ParseAbsoluteGeometry(image_info->size,&image->extract_info);
      image->columns=image->extract_info.width;
      image->rows=image->extract_info.height;
      image->offset=image->extract_info.x;
      image->extract_info.x=0;
      image->extract_info.y=0;
    }
  if (image_info->extract != (char *) NULL)
    {
      RectangleInfo
        geometry;

      (void) memset(&geometry,0,sizeof(geometry));
      flags=ParseAbsoluteGeometry(image_info->extract,&geometry);
      if (((flags & XValue) != 0) || ((flags & YValue) != 0))
        {
          image->extract_info=geometry;
          Swap(image->columns,image->extract_info.width);
          Swap(image->rows,image->extract_info.height);
        }
    }
  image->compression=image_info->compression;
  image->quality=image_info->quality;
  image->endian=image_info->endian;
  image->interlace=image_info->interlace;
  image->units=image_info->units;
  if (image_info->density != (char *) NULL)
    {
      GeometryInfo
        geometry_info;

      flags=ParseGeometry(image_info->density,&geometry_info);
      if ((flags & RhoValue) != 0)
        image->resolution.x=geometry_info.rho;
      image->resolution.y=image->resolution.x;
      if ((flags & SigmaValue) != 0)
        image->resolution.y=geometry_info.sigma;
    }
  if (image_info->page != (char *) NULL)
    {
      char
        *geometry;

      image->page=image->extract_info;
      geometry=GetPageGeometry(image_info->page);
      (void) ParseAbsoluteGeometry(geometry,&image->page);
      geometry=DestroyString(geometry);
    }
  if (image_info->depth != 0)
    image->depth=image_info->depth;
  image->dither=image_info->dither;
  image->matte_color=image_info->matte_color;
  image->background_color=image_info->background_color;
  image->border_color=image_info->border_color;
  image->transparent_color=image_info->transparent_color;
  image->ping=image_info->ping;
  image->progress_monitor=image_info->progress_monitor;
  image->client_data=image_info->client_data;
  if (image_info->cache != (void *) NULL)
    ClonePixelCacheMethods(image->cache,image_info->cache);
  /*
    Set all global options that map to per-image settings.
  */
  (void) SyncImageSettings(image_info,image,exception);
  /*
    Global options that are only set for new images.
  */
  option=GetImageOption(image_info,"delay");
  if (option != (const char *) NULL)
    {
      GeometryInfo
        geometry_info;

      flags=ParseGeometry(option,&geometry_info);
      if ((flags & GreaterValue) != 0)
        {
          if (image->delay > (size_t) floor(geometry_info.rho+0.5))
            image->delay=(size_t) floor(geometry_info.rho+0.5);
        }
      else
        if ((flags & LessValue) != 0)
          {
            if (image->delay < (size_t) floor(geometry_info.rho+0.5))
              image->ticks_per_second=(ssize_t) floor(geometry_info.sigma+0.5);
          }
        else
          image->delay=(size_t) floor(geometry_info.rho+0.5);
      if ((flags & SigmaValue) != 0)
        image->ticks_per_second=(ssize_t) floor(geometry_info.sigma+0.5);
    }
  option=GetImageOption(image_info,"dispose");
  if (option != (const char *) NULL)
    image->dispose=(DisposeType) ParseCommandOption(MagickDisposeOptions,
      MagickFalse,option);
  return(image);
}


MagickExport MagickStatusType ParsePageGeometry(const Image *image,
  const char *geometry,RectangleInfo *region_info,ExceptionInfo *exception)
{
  MagickStatusType
    flags;

  SetGeometry(image,region_info);
  if (image->page.width != 0)
    region_info->width=image->page.width;
  if (image->page.height != 0)
    region_info->height=image->page.height;
  flags=ParseAbsoluteGeometry(geometry,region_info);
  if (flags == NoValue)
    {
      (void) ThrowMagickException(exception,GetMagickModule(),OptionError,
        "InvalidGeometry","`%s'",geometry);
      return(flags);
    }
  if ((flags & PercentValue) != 0)
    {
      region_info->width=image->columns;
      region_info->height=image->rows;
    }
  flags=ParseMetaGeometry(geometry,&region_info->x,&region_info->y,
    &region_info->width,&region_info->height);
  if ((((flags & WidthValue) != 0) || ((flags & HeightValue) != 0)) &&
      (((flags & PercentValue) != 0) || ((flags & SeparatorValue) == 0)))
    {
      if ((flags & WidthValue) == 0)
        region_info->width=region_info->height;
      if ((flags & HeightValue) == 0)
        region_info->height=region_info->width;
    }
  return(flags);
}


static MagickBooleanType WritePALMImage(const ImageInfo *image_info,
  Image *image,ExceptionInfo *exception)
{
  MagickBooleanType
    status;

  MagickOffsetType
    currentOffset,
    offset,
    scene;

  MagickSizeType
    cc;

  PixelInfo
    transpix;

  QuantizeInfo
    *quantize_info;

  register ssize_t
    x;

  register const Quantum
    *p;

  register Quantum
    *q;

  ssize_t
    y;

  size_t
    count,
    bits_per_pixel,
    bytes_per_row,
    imageListLength,
    nextDepthOffset,
    one;

  unsigned char
    bit,
    byte,
    color,
    *last_row,
    *one_row,
    *ptr,
    version;

  unsigned int
    transparentIndex;

  unsigned short
    color16,
    flags;

  /*
    Open output image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  status=OpenBlob(image_info,image,WriteBinaryBlobMode,exception);
  if (status == MagickFalse)
    return(status);
  quantize_info=AcquireQuantizeInfo(image_info);
  flags=0;
  currentOffset=0;
  transparentIndex=0;
  transpix.red=0.0;
  transpix.green=0.0;
  transpix.blue=0.0;
  transpix.alpha=0.0;
  one=1;
  version=0;
  scene=0;
  imageListLength=GetImageListLength(image);
  do
  {
    (void) TransformImageColorspace(image,sRGBColorspace,exception);
    count=GetNumberColors(image,NULL,exception);
    for (bits_per_pixel=1; (one << bits_per_pixel) < count; bits_per_pixel*=2) ;
    if (bits_per_pixel > 16)
      bits_per_pixel=16;
    else
      if (bits_per_pixel < 16)
        (void) TransformImageColorspace(image,image->colorspace,exception);
    if (bits_per_pixel < 8)
      {
        (void) TransformImageColorspace(image,GRAYColorspace,exception);
        (void) SetImageType(image,PaletteType,exception);
        (void) SortColormapByIntensity(image,exception);
      }
    if ((image->storage_class == PseudoClass) && (image->colors > 256))
      (void) SetImageStorageClass(image,DirectClass,exception);
    if (image->storage_class == PseudoClass)
      flags|=PALM_HAS_COLORMAP_FLAG;
    else
      flags|=PALM_IS_DIRECT_COLOR;
    (void) WriteBlobMSBShort(image,(unsigned short) image->columns); /* width */
    (void) WriteBlobMSBShort(image,(unsigned short) image->rows);  /* height */
    bytes_per_row=((image->columns+(16/bits_per_pixel-1))/(16/
      bits_per_pixel))*2;
    (void) WriteBlobMSBShort(image,(unsigned short) bytes_per_row);
    if ((image_info->compression == RLECompression) ||
        (image_info->compression == FaxCompression))
      flags|=PALM_IS_COMPRESSED_FLAG;
    (void) WriteBlobMSBShort(image, flags);
    (void) WriteBlobByte(image,(unsigned char) bits_per_pixel);
    if (bits_per_pixel > 1)
      version=1;
    if ((image_info->compression == RLECompression) ||
        (image_info->compression == FaxCompression))
      version=2;
    (void) WriteBlobByte(image,version);
    (void) WriteBlobMSBShort(image,0);  /* nextDepthOffset */
    (void) WriteBlobByte(image,(unsigned char) transparentIndex);
    if (image_info->compression == RLECompression)
      (void) WriteBlobByte(image,PALM_COMPRESSION_RLE);
    else
      if (image_info->compression == FaxCompression)
        (void) WriteBlobByte(image,PALM_COMPRESSION_SCANLINE);
      else
        (void) WriteBlobByte(image,PALM_COMPRESSION_NONE);
    (void) WriteBlobMSBShort(image,0);  /* reserved */
    offset=16;
    if (bits_per_pixel == 16)
      {
        (void) WriteBlobByte(image,5);  /* # of bits of red */
        (void) WriteBlobByte(image,6);  /* # of bits of green */
        (void) WriteBlobByte(image,5);  /* # of bits of blue */
        (void) WriteBlobByte(image,0);  /* reserved by Palm */
        (void) WriteBlobMSBLong(image,0);  /* no transparent color, YET */
        offset+=8;
      }
    if (bits_per_pixel == 8)
      {
        if (flags & PALM_HAS_COLORMAP_FLAG)  /* Write out colormap */
          {
            quantize_info->dither_method=IdentifyPaletteImage(image,exception)
              == MagickFalse ? RiemersmaDitherMethod : NoDitherMethod;
            quantize_info->number_colors=image->colors;
            (void) QuantizeImage(quantize_info,image,exception);
            (void) WriteBlobMSBShort(image,(unsigned short) image->colors);
            for (count = 0; count < image->colors; count++)
            {
              (void) WriteBlobByte(image,(unsigned char) count);
              (void) WriteBlobByte(image,ScaleQuantumToChar(ClampToQuantum(
                image->colormap[count].red)));
              (void) WriteBlobByte(image,ScaleQuantumToChar(ClampToQuantum(
                image->colormap[count].green)));
              (void) WriteBlobByte(image,ScaleQuantumToChar(ClampToQuantum(
                image->colormap[count].blue)));
            }
            offset+=2+count*4;
          }
      else  /* Map colors to Palm standard colormap */
        {
          Image
            *affinity_image;

          affinity_image=ConstituteImage(256,1,"RGB",CharPixel,&PalmPalette,
            exception);
          (void) TransformImageColorspace(affinity_image,
            affinity_image->colorspace,exception);
          (void) RemapImage(quantize_info,image,affinity_image,exception);
          for (y=0; y < (ssize_t) image->rows; y++)
          {
            q=GetAuthenticPixels(image,0,y,image->columns,1,exception);
            if (q == (Quantum *) NULL)
              break;
            for (x=0; x < (ssize_t) image->columns; x++)
            {
              SetPixelIndex(image,(Quantum) FindColor(&image->colormap[(ssize_t)
                GetPixelIndex(image,q)]),q);
              q+=GetPixelChannels(image);
            }
          }
          affinity_image=DestroyImage(affinity_image);
        }
      }
    if (flags & PALM_IS_COMPRESSED_FLAG)
      (void) WriteBlobMSBShort(image,0);  /* fill in size later */
    last_row=(unsigned char *) NULL;
    if (image_info->compression == FaxCompression)
      {
        last_row=(unsigned char *) AcquireQuantumMemory(bytes_per_row+256,
          sizeof(*last_row));
        if (last_row == (unsigned char *) NULL)
          {
            quantize_info=DestroyQuantizeInfo(quantize_info);
            ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
          }
      }
    one_row=(unsigned char *) AcquireQuantumMemory(bytes_per_row+256,
      sizeof(*one_row));
    if (one_row == (unsigned char *) NULL)
      {
        if (last_row != (unsigned char *) NULL)
          last_row=(unsigned char *) RelinquishMagickMemory(last_row);
        quantize_info=DestroyQuantizeInfo(quantize_info);
        ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
      }
    for (y=0; y < (ssize_t) image->rows; y++)
    {
      ptr=one_row;
      (void) memset(ptr,0,bytes_per_row);
      p=GetVirtualPixels(image,0,y,image->columns,1,exception);
      if (p == (const Quantum *) NULL)
        break;
      if (bits_per_pixel == 16)
        {
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            color16=(unsigned short) ((((31*(ssize_t) GetPixelRed(image,p))/
              (ssize_t) QuantumRange) << 11) | (((63*(ssize_t)
              GetPixelGreen(image,p))/(ssize_t) QuantumRange) << 5) |
              ((31*(ssize_t) GetPixelBlue(image,p))/(ssize_t) QuantumRange));
            if (GetPixelAlpha(image,p) == (Quantum) TransparentAlpha)
              {
                transpix.red=(MagickRealType) GetPixelRed(image,p);
                transpix.green=(MagickRealType) GetPixelGreen(image,p);
                transpix.blue=(MagickRealType) GetPixelBlue(image,p);
                transpix.alpha=(MagickRealType) GetPixelAlpha(image,p);
                flags|=PALM_HAS_TRANSPARENCY_FLAG;
              }
            *ptr++=(unsigned char) ((color16 >> 8) & 0xff);
            *ptr++=(unsigned char) (color16 & 0xff);
            p+=GetPixelChannels(image);
          }
        }
      else
        {
          byte=0x00;
          bit=(unsigned char) (8-bits_per_pixel);
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            if (bits_per_pixel >= 8)
              color=(unsigned char) GetPixelIndex(image,p);
            else
              color=(unsigned char) (GetPixelIndex(image,p)*
                ((one << bits_per_pixel)-1)/MagickMax(1*image->colors-1,1));
            byte|=color << bit;
            if (bit != 0)
              bit-=(unsigned char) bits_per_pixel;
            else
              {
                *ptr++=byte;
                byte=0x00;
                bit=(unsigned char) (8-bits_per_pixel);
              }
            p+=GetPixelChannels(image);
          }
          if ((image->columns % (8/bits_per_pixel)) != 0)
            *ptr++=byte;
        }
      if (image_info->compression == RLECompression)
        {
          x=0;
          while (x < (ssize_t) bytes_per_row)
          {
            byte=one_row[x];
            count=1;
            while ((one_row[++x] == byte) && (count < 255) &&
                   (x < (ssize_t) bytes_per_row))
              count++;
            (void) WriteBlobByte(image,(unsigned char) count);
            (void) WriteBlobByte(image,(unsigned char) byte);
          }
        }
      else
        if (image_info->compression == FaxCompression)
          {
            char
              tmpbuf[8],
              *tptr;

            for (x = 0;  x < (ssize_t) bytes_per_row;  x += 8)
            {
              tptr = tmpbuf;
              for (bit=0, byte=0; bit < (unsigned char) MagickMin(8,(ssize_t) bytes_per_row-x); bit++)
              {
                if ((y == 0) || (last_row[x + bit] != one_row[x + bit]))
                  {
                    byte |= (1 << (7 - bit));
                    *tptr++ = (char) one_row[x + bit];
                  }
              }
              (void) WriteBlobByte(image, byte);
              (void) WriteBlob(image,tptr-tmpbuf,(unsigned char *) tmpbuf);
            }
            (void) memcpy(last_row,one_row,bytes_per_row);
          }
        else
          (void) WriteBlob(image,bytes_per_row,one_row);
      }
    if (flags & PALM_HAS_TRANSPARENCY_FLAG)
      {
        offset=SeekBlob(image,currentOffset+6,SEEK_SET);
        (void) WriteBlobMSBShort(image,flags);
        offset=SeekBlob(image,currentOffset+12,SEEK_SET);
        (void) WriteBlobByte(image,(unsigned char) transparentIndex);  /* trans index */
      }
    if (bits_per_pixel == 16)
      {
        offset=SeekBlob(image,currentOffset+20,SEEK_SET);
        (void) WriteBlobByte(image,0);  /* reserved by Palm */
        (void) WriteBlobByte(image,(unsigned char) ((31*transpix.red)/
          QuantumRange));
        (void) WriteBlobByte(image,(unsigned char) ((63*transpix.green)/
          QuantumRange));
        (void) WriteBlobByte(image,(unsigned char) ((31*transpix.blue)/
          QuantumRange));
      }
    if (flags & PALM_IS_COMPRESSED_FLAG)  /* fill in size now */
      {
        offset=SeekBlob(image,currentOffset+offset,SEEK_SET);
        (void) WriteBlobMSBShort(image,(unsigned short) (GetBlobSize(image)-
          currentOffset-offset));
      }
    if (one_row != (unsigned char *) NULL)
      one_row=(unsigned char *) RelinquishMagickMemory(one_row);
    if (last_row != (unsigned char *) NULL)
      last_row=(unsigned char *) RelinquishMagickMemory(last_row);
    if (GetNextImageInList(image) == (Image *) NULL)
      break;
    /* padding to 4 byte word */
    for (cc=(GetBlobSize(image)) % 4; cc > 0; cc--)
      (void) WriteBlobByte(image,0);
    /* write nextDepthOffset and return to end of image */
    offset=SeekBlob(image,currentOffset+10,SEEK_SET);
    nextDepthOffset=(size_t) ((GetBlobSize(image)-currentOffset)/4);
    (void) WriteBlobMSBShort(image,(unsigned short) nextDepthOffset);
    currentOffset=(MagickOffsetType) GetBlobSize(image);
    offset=SeekBlob(image,currentOffset,SEEK_SET);
    image=SyncNextImageInList(image);
    status=SetImageProgress(image,SaveImagesTag,scene++,imageListLength);
    if (status == MagickFalse)
      break;
  } while (image_info->adjoin != MagickFalse);
  quantize_info=DestroyQuantizeInfo(quantize_info);
  (void) CloseBlob(image);
  return(MagickTrue);
}


static double Blackman(const double x,
  const ResizeFilter *magick_unused(resize_filter))
{
  /*
    Blackman: 2nd order cosine windowing function:
      0.42 + 0.5 cos(pi x) + 0.08 cos(2pi x)

    Refactored by Chantal Racette and Nicolas Robidoux to one trig call and
    five flops.
  */
  const double cosine = cos((double) (MagickPI*x));
  magick_unreferenced(resize_filter);
  return(0.34+cosine*(0.5+cosine*0.16));
}


static void InitializeHistogram(const Image *image,ssize_t **histogram,
  ExceptionInfo *exception)
{
  register const Quantum
    *p;

  register ssize_t
    i,
    x;

  ssize_t
    y;

  /*
    Initialize histogram.
  */
  for (i=0; i <= 255; i++)
  {
    histogram[Red][i]=0;
    histogram[Green][i]=0;
    histogram[Blue][i]=0;
  }
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    p=GetVirtualPixels(image,0,y,image->columns,1,exception);
    if (p == (const Quantum *) NULL)
      break;
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      histogram[Red][(ssize_t) ScaleQuantumToChar(GetPixelRed(image,p))]++;
      histogram[Green][(ssize_t) ScaleQuantumToChar(GetPixelGreen(image,p))]++;
      histogram[Blue][(ssize_t) ScaleQuantumToChar(GetPixelBlue(image,p))]++;
      p+=GetPixelChannels(image);
    }
  }
}


static void ScaleSpace(const ssize_t *histogram,const double tau,
  double *scale_histogram)
{
  double
    alpha,
    beta,
    *gamma,
    sum;

  register ssize_t
    u,
    x;

  gamma=(double *) AcquireQuantumMemory(256,sizeof(*gamma));
  if (gamma == (double *) NULL)
    ThrowFatalException(ResourceLimitFatalError,
      "UnableToAllocateGammaMap");
  alpha=PerceptibleReciprocal(tau*sqrt(2.0*MagickPI));
  beta=(-1.0*PerceptibleReciprocal(2.0*tau*tau));
  for (x=0; x <= 255; x++)
    gamma[x]=0.0;
  for (x=0; x <= 255; x++)
  {
    gamma[x]=exp((double) beta*x*x);
    if (gamma[x] < MagickEpsilon)
      break;
  }
  for (x=0; x <= 255; x++)
  {
    sum=0.0;
    for (u=0; u <= 255; u++)
      sum+=(double) histogram[u]*gamma[MagickAbsoluteValue(x-u)];
    scale_histogram[x]=alpha*sum;
  }
  gamma=(double *) RelinquishMagickMemory(gamma);
}


static void DerivativeHistogram(const double *histogram,
  double *derivative)
{
  register ssize_t
    i,
    n;

  /*
    Compute endpoints using second order polynomial interpolation.
  */
  n=255;
  derivative[0]=(-1.5*histogram[0]+2.0*histogram[1]-0.5*histogram[2]);
  derivative[n]=(0.5*histogram[n-2]-2.0*histogram[n-1]+1.5*histogram[n]);
  /*
    Compute derivative using central differencing.
  */
  for (i=1; i < n; i++)
    derivative[i]=(histogram[i+1]-histogram[i-1])/2.0;
  return;
}


static inline unsigned int ScaleQuantumToLong(const Quantum quantum)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((unsigned int) (16843009UL*quantum));
#else
  if ((IsNaN(quantum) != MagickFalse) || (quantum <= 0.0))
    return(0U);
  if ((16843009.0*quantum) >= 4294967295.0)
    return(4294967295UL);
  return((unsigned int) (16843009.0*quantum+0.5));
#endif
}


static inline unsigned int ScaleQuantumToMap(const Quantum quantum)
{
  if ((IsNaN(quantum) != MagickFalse) || (quantum <= 0.0))
    return(0U);
  if ((quantum/281479271743489.0) >= MaxMap)
    return((unsigned int) MaxMap);
  return((unsigned int) (quantum/281479271743489.0+0.5));
}


static int IntensityCompare(const void *x,const void *y)
{
  double
    intensity;

  PixelInfo
    *color_1,
    *color_2;

  color_1=(PixelInfo *) x;
  color_2=(PixelInfo *) y;
  intensity=GetPixelInfoIntensity((const Image *) NULL,color_1)-
    GetPixelInfoIntensity((const Image *) NULL,color_2);
  if (intensity > (double) INT_MAX)
    intensity=(double) INT_MAX;
  if (intensity < (double) INT_MIN)
    intensity=(double) INT_MIN;
  return((int) intensity);
}


MagickExport const char *GetStringInfoName(const StringInfo *string_info)
{
  assert(string_info != (StringInfo *) NULL);
  assert(string_info->signature == MagickCoreSignature);
  return(string_info->name);
}


MagickExport char *CloneString(char **destination,const char *source)
{
  size_t
    length;

  assert(destination != (char **) NULL);
  if (source == (const char *) NULL)
    {
      if (*destination != (char *) NULL)
        *destination=DestroyString(*destination);
      return(*destination);
    }
  if (*destination == (char *) NULL)
    {
      *destination=AcquireString(source);
      return(*destination);
    }
  length=strlen(source);
  if (~length < MagickPathExtent)
    ThrowFatalException(ResourceLimitFatalError,"UnableToAcquireString");
  *destination=(char *) ResizeQuantumMemory(*destination,length+
    MagickPathExtent,sizeof(**destination));
  if (*destination == (char *) NULL)
    ThrowFatalException(ResourceLimitFatalError,"UnableToAcquireString");
  if (length != 0)
    (void) memcpy(*destination,source,length*sizeof(**destination));
  (*destination)[length]='\0';
  return(*destination);
}


MagickExport char **StringToStrings(const char *text,size_t *count)
{
  char
    **textlist;

  register const char
    *p;

  register ssize_t
    i;

  size_t
    lines;

  if (text == (char *) NULL)
    {
      if (count != (size_t *) NULL)
        *count=0;
      return((char **) NULL);
    }
  for (p=text; *p != '\0'; p++)
    if (((int) ((unsigned char) *p) < 32) &&
        (isspace((int) ((unsigned char) *p)) == 0))
      break;
  if (*p == '\0')
    {
      register const char
        *q;

      /*
        Convert string to an ASCII list.
      */
      lines=1;
      for (p=text; *p != '\0'; p++)
        if (*p == '\n')
          lines++;
      textlist=(char **) AcquireQuantumMemory((size_t) lines+1UL,
        sizeof(*textlist));
      if (textlist == (char **) NULL)
        ThrowFatalException(ResourceLimitFatalError,"UnableToConvertText");
      p=text;
      for (i=0; i < (ssize_t) lines; i++)
      {
        for (q=p; *q != '\0'; q++)
          if ((*q == '\r') || (*q == '\n'))
            break;
        textlist[i]=(char *) AcquireQuantumMemory((size_t) (q-p)+1,
          sizeof(**textlist));
        if (textlist[i] == (char *) NULL)
          ThrowFatalException(ResourceLimitFatalError,"UnableToConvertText");
        (void) memcpy(textlist[i],p,(size_t) (q-p));
        textlist[i][q-p]='\0';
        if (*q == '\r')
          q++;
        p=q+1;
      }
    }
  else
    {
      char
        hex_string[MagickPathExtent];

      register char
        *q;

      register ssize_t
        j;

      /*
        Convert string to a HEX list.
      */
      lines=(size_t) (strlen(text)/CharsPerLine)+1;
      textlist=(char **) AcquireQuantumMemory((size_t) lines+1UL,
        sizeof(*textlist));
      if (textlist == (char **) NULL)
        ThrowFatalException(ResourceLimitFatalError,"UnableToConvertText");
      p=text;
      for (i=0; i < (ssize_t) lines; i++)
      {
        textlist[i]=(char *) AcquireQuantumMemory(2UL*MagickPathExtent,
          sizeof(**textlist));
        if (textlist[i] == (char *) NULL)
          ThrowFatalException(ResourceLimitFatalError,"UnableToConvertText");
        (void) FormatLocaleString(textlist[i],MagickPathExtent,"0x%08lx: ",
          (long) (CharsPerLine*i));
        q=textlist[i]+strlen(textlist[i]);
        for (j=1; j <= (ssize_t) MagickMin(strlen(p),CharsPerLine); j++)
        {
          (void) FormatLocaleString(hex_string,MagickPathExtent,"%02x",*(p+j));
          (void) CopyMagickString(q,hex_string,MagickPathExtent);
          q+=2;
          if ((j % 0x04) == 0)
            *q++=' ';
        }
        for ( ; j <= CharsPerLine; j++)
        {
          *q++=' ';
          *q++=' ';
          if ((j % 0x04) == 0)
            *q++=' ';
        }
        *q++=' ';
        for (j=1; j <= (ssize_t) MagickMin(strlen(p),CharsPerLine); j++)
        {
          if (isprint((int) ((unsigned char) *p)) != 0)
            *q++=(*p);
          else
            *q++='-';
          p++;
        }
        *q='\0';
        textlist[i]=(char *) ResizeQuantumMemory(textlist[i],q-textlist[i]+1,
          sizeof(**textlist));
        if (textlist[i] == (char *) NULL)
          ThrowFatalException(ResourceLimitFatalError,"UnableToConvertText");
      }
    }
  if (count != (size_t *) NULL)
    *count=lines;
  textlist[i]=(char *) NULL;
  return(textlist);
}


static int wsrep_create_sst_user(bool initialize_thread, const char *password) {
  int err = 0;
  int const auth_len = 512;
  char auth_buf[auth_len];
  MYSQL_SESSION session = NULL;

  // This array is filled with pairs of entries
  // The first entry is the actual query to be run
  // The second entry is the string to be displayed if the query fails
  //  (this can be NULL, in which case the actual query will be used)
  const char *cmds[] = {
    "SET SESSION sql_log_bin = OFF;",
    nullptr,
    "DROP USER IF EXISTS 'mysql.pxc.sst.user'@localhost;",
    nullptr,
    "CREATE USER 'mysql.pxc.sst.user'@localhost "
    " IDENTIFIED BY '%s' ACCOUNT LOCK;",
    "CREATE USER mysql.pxc.sst.user IDENTIFIED WITH * BY * ACCOUNT LOCK",
  /*
    This is the code that uses the mysql.pxc.sst.role
    However there is a bug in 8.0.15 where the "GRANT CREATE ON DBNAME.*" when
    used in a role, does not allow the user with the role to create a database.
    So we have to explicitly grant the privileges.
  */
#if 0
    "GRANT 'mysql.pxc.sst.role'@localhost TO 'mysql.pxc.sst.user'@localhost;", nullptr,
    "SET DEFAULT ROLE 'mysql.pxc.sst.role'@localhost to 'mysql.pxc.sst.user'@localhost;", nullptr,
#else
    /*
      Explicit privileges needed to run XtraBackup.  This is only used due
      to the bug in 8.0.15 described above.
    */
    "GRANT BACKUP_ADMIN, LOCK TABLES, PROCESS, RELOAD, REPLICATION CLIENT, "
    "SUPER ON *.* TO 'mysql.pxc.sst.user'@localhost;",
    nullptr,
    "GRANT CREATE, INSERT, SELECT ON PERCONA_SCHEMA.xtrabackup_history TO "
    "'mysql.pxc.sst.user'@localhost;",
    nullptr,
    "GRANT SELECT ON performance_schema.* TO 'mysql.pxc.sst.user'@localhost;",
    nullptr,
    "GRANT CREATE USER ON *.* to 'mysql.pxc.sst.user'@localhost;",
    nullptr,
    "GRANT CREATE ON PERCONA_SCHEMA.* to 'mysql.pxc.sst.user'@localhost;",
    nullptr,
#endif

    "ALTER USER 'mysql.pxc.sst.user'@localhost ACCOUNT UNLOCK;",
    nullptr,
    nullptr,
    nullptr
  };

  wsrep_allow_server_session = true;
  session = setup_server_session(initialize_thread);
  if (!session) {
    wsrep_allow_server_session = false;
    return ECANCELED;
  }

  for (int index = 0; !err && cmds[index]; index += 2) {
    int ret;
    ret = snprintf(auth_buf, auth_len, cmds[index], password);
    if (ret < 0 || ret >= auth_len) {
      WSREP_ERROR("wsrep_create_sst_user() : snprintf() failed: %d", ret);
      err = (ret < 0 ? ret : -EMSGSIZE);
      break;
    }
    // Ignore errors: also ignored in the boostrap code
    err = server_session_execute(session, auth_buf, cmds[index + 1], true);
  }

  // Overwrite query (clear out any sensitive data)
  ::memset(auth_buf, 0, auth_len);

  cleanup_server_session(session, initialize_thread);
  wsrep_allow_server_session = false;
  return err;
}


static void cleanup_server_session(MYSQL_SESSION session,
                                   bool initialize_thread) {
  srv_session_close(session);
  if (initialize_thread) srv_session_deinit_thread();
}


static enum loglevel string_to_loglevel(const char *s) {
  if (strncmp(s, "ERR:", 4) == 0)
    return ERROR_LEVEL;
  else if (strncmp(s, "WRN:", 4) == 0)
    return WARNING_LEVEL;
  else if (strncmp(s, "INF:", 4) == 0)
    return INFORMATION_LEVEL;
  else if (strncmp(s, "DBG:", 4) == 0)
    return INFORMATION_LEVEL;
  else
    return SYSTEM_LEVEL;
}


bool wsrep_sst_auth_check (sys_var *self, THD* thd, set_var* var)
{
    return 0;
}


bool wsrep_sst_wait ()
{
  if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort();
  while (!sst_complete)
  {
    WSREP_INFO("Waiting for SST to complete.");
    mysql_cond_wait (&COND_wsrep_sst, &LOCK_wsrep_sst);
  }

  if (local_seqno >= 0)
  {
    WSREP_INFO("SST complete, seqno: %lld", (long long) local_seqno);
  }
  else
  {
    WSREP_ERROR("SST failed: %d (%s)",
                int(-local_seqno), strerror(-local_seqno));
  }

  mysql_mutex_unlock (&LOCK_wsrep_sst);

  return (local_seqno >= 0);
}


static int sst_donate_mysqldump (const char*         addr,
                                 const wsrep_uuid_t* uuid,
                                 const char*         uuid_str,
                                 wsrep_seqno_t       seqno,
                                 bool                bypass,
                                 char**              env) // carries auth info
{
  int const cmd_len= 4096;
  wsp::string  cmd_str(cmd_len);

  if (!cmd_str())
  {
    WSREP_ERROR("sst_donate_mysqldump(): "
                "could not allocate cmd buffer of %d bytes", cmd_len);
    return -ENOMEM;
  }

  if (!bypass && wsrep_sst_donor_rejects_queries) sst_reject_queries(TRUE);

  int ret= snprintf (cmd_str(), cmd_len,
                     "wsrep_sst_mysqldump "
                     WSREP_SST_OPT_ADDR" '%s' "
                     WSREP_SST_OPT_LPORT" '%u' "
                     WSREP_SST_OPT_SOCKET" '%s' "
                     WSREP_SST_OPT_CONF" '%s' "
                     WSREP_SST_OPT_GTID" '%s:%lld'"
                     "%s",
                     addr, mysqld_port, mysqld_unix_port,
                     wsrep_defaults_file, uuid_str,
                     (long long)seqno, bypass ? " " WSREP_SST_OPT_BYPASS : "");

  if (ret < 0 || ret >= cmd_len)
  {
    WSREP_ERROR("sst_donate_mysqldump(): snprintf() failed: %d", ret);
    return (ret < 0 ? ret : -EMSGSIZE);
  }

  WSREP_DEBUG("Running: '%s'", cmd_str());

  ret= sst_run_shell (cmd_str(), env, 3);

  wsrep_gtid_t const state_id = { *uuid, (ret ? WSREP_SEQNO_UNDEFINED : seqno)};

  wsrep->sst_sent (wsrep, &state_id, ret);

  return ret;
}


void wsrep_sst_complete (const wsrep_uuid_t* sst_uuid,
                         wsrep_seqno_t       sst_seqno,
                         bool                needed)
{
  if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort();
  if (!sst_complete)
  {
    sst_complete = true;
    sst_needed   = needed;
    local_uuid   = *sst_uuid;
    local_seqno  = sst_seqno;
    mysql_cond_signal (&COND_wsrep_sst);
  }
  else
  {
    /* This can happen when called from wsrep_synced_cb().
       At the moment there is no way to check there
       if main thread is still waiting for signal,
       so wsrep_sst_complete() is called from there
       each time wsrep_ready changes from FALSE -> TRUE.
    */
    WSREP_DEBUG("Nobody is waiting for SST.");
  }
  mysql_mutex_unlock (&LOCK_wsrep_sst);
}


static void perf_swevent_init_cpu(unsigned int cpu)
{
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);

	mutex_lock(&swhash->hlist_mutex);
	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
		struct swevent_hlist *hlist;

		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
		WARN_ON(!hlist);
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
	}
	mutex_unlock(&swhash->hlist_mutex);
}


static int perf_event_bpf_match(struct perf_event *event)
{
	return event->attr.bpf_event;
}


static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
{
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
	int maj = 0, min = 0;
	u64 ino = 0, gen = 0;
	u32 prot = 0, flags = 0;
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
	char *name;

	if (vma->vm_flags & VM_READ)
		prot |= PROT_READ;
	if (vma->vm_flags & VM_WRITE)
		prot |= PROT_WRITE;
	if (vma->vm_flags & VM_EXEC)
		prot |= PROT_EXEC;

	if (vma->vm_flags & VM_MAYSHARE)
		flags = MAP_SHARED;
	else
		flags = MAP_PRIVATE;

	if (vma->vm_flags & VM_DENYWRITE)
		flags |= MAP_DENYWRITE;
	if (vma->vm_flags & VM_MAYEXEC)
		flags |= MAP_EXECUTABLE;
	if (vma->vm_flags & VM_LOCKED)
		flags |= MAP_LOCKED;
	if (is_vm_hugetlb_page(vma))
		flags |= MAP_HUGETLB;

	if (file) {
		struct inode *inode;
		dev_t dev;

		buf = kmalloc(PATH_MAX, GFP_KERNEL);
		if (!buf) {
			name = "//enomem";
			goto cpy_name;
		}
		/*
		 * d_path() works from the end of the rb backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		name = file_path(file, buf, PATH_MAX - sizeof(u64));
		if (IS_ERR(name)) {
			name = "//toolong";
			goto cpy_name;
		}
		inode = file_inode(vma->vm_file);
		dev = inode->i_sb->s_dev;
		ino = inode->i_ino;
		gen = inode->i_generation;
		maj = MAJOR(dev);
		min = MINOR(dev);

		goto got_name;
	} else {
		if (vma->vm_ops && vma->vm_ops->name) {
			name = (char *) vma->vm_ops->name(vma);
			if (name)
				goto cpy_name;
		}

		name = (char *)arch_vma_name(vma);
		if (name)
			goto cpy_name;

		if (vma->vm_start <= vma->vm_mm->start_brk &&
				vma->vm_end >= vma->vm_mm->brk) {
			name = "[heap]";
			goto cpy_name;
		}
		if (vma->vm_start <= vma->vm_mm->start_stack &&
				vma->vm_end >= vma->vm_mm->start_stack) {
			name = "[stack]";
			goto cpy_name;
		}

		name = "//anon";
		goto cpy_name;
	}

cpy_name:
	strlcpy(tmp, name, sizeof(tmp));
	name = tmp;
got_name:
	/*
	 * Since our buffer works in 8 byte units we need to align our string
	 * size to a multiple of 8. However, we must guarantee the tail end is
	 * zero'd out to avoid leaking random bits to userspace.
	 */
	size = strlen(name)+1;
	while (!IS_ALIGNED(size, sizeof(u64)))
		name[size++] = '\0';

	mmap_event->file_name = name;
	mmap_event->file_size = size;
	mmap_event->maj = maj;
	mmap_event->min = min;
	mmap_event->ino = ino;
	mmap_event->ino_generation = gen;
	mmap_event->prot = prot;
	mmap_event->flags = flags;

	if (!(vma->vm_flags & VM_EXEC))
		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;

	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;

	perf_iterate_sb(perf_event_mmap_output,
		       mmap_event,
		       NULL);

	kfree(buf);
}


static int perf_event_check_period(struct perf_event *event, u64 value)
{
	return event->pmu->check_period(event, value);
}


void perf_event_task_tick(void)
{
	struct list_head *head = this_cpu_ptr(&active_ctx_list);
	struct perf_event_context *ctx, *tmp;
	int throttled;

	lockdep_assert_irqs_disabled();

	__this_cpu_inc(perf_throttled_seq);
	throttled = __this_cpu_xchg(perf_throttled_count, 0);
	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);

	list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
		perf_adjust_freq_unthr_context(ctx, throttled);
}


static void _free_event(struct perf_event *event)
{
	irq_work_sync(&event->pending);

	unaccount_event(event);

	security_perf_event_free(event);

	if (event->rb) {
		/*
		 * Can happen when we close an event with re-directed output.
		 *
		 * Since we have a 0 refcount, perf_mmap_close() will skip
		 * over us; possibly making our ring_buffer_put() the last.
		 */
		mutex_lock(&event->mmap_mutex);
		ring_buffer_attach(event, NULL);
		mutex_unlock(&event->mmap_mutex);
	}

	if (is_cgroup_event(event))
		perf_detach_cgroup(event);

	if (!event->parent) {
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
			put_callchain_buffers();
	}

	perf_event_free_bpf_prog(event);
	perf_addr_filters_splice(event, NULL);
	kfree(event->addr_filter_ranges);

	if (event->destroy)
		event->destroy(event);

	/*
	 * Must be after ->destroy(), due to uprobe_perf_close() using
	 * hw.target.
	 */
	if (event->hw.target)
		put_task_struct(event->hw.target);

	/*
	 * perf_event_free_task() relies on put_ctx() being 'last', in particular
	 * all task references must be cleaned up.
	 */
	if (event->ctx)
		put_ctx(event->ctx);

	exclusive_event_destroy(event);
	module_put(event->pmu->module);

	call_rcu(&event->rcu_head, free_event_rcu);
}


long perf_pmu_snapshot_aux(struct perf_buffer *rb,
			   struct perf_event *event,
			   struct perf_output_handle *handle,
			   unsigned long size)
{
	unsigned long flags;
	long ret;

	/*
	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
	 * paths. If we start calling them in NMI context, they may race with
	 * the IRQ ones, that is, for example, re-starting an event that's just
	 * been stopped, which is why we're using a separate callback that
	 * doesn't change the event state.
	 *
	 * IRQs need to be disabled to prevent IPIs from racing with us.
	 */
	local_irq_save(flags);
	/*
	 * Guard against NMI hits inside the critical section;
	 * see also perf_prepare_sample_aux().
	 */
	WRITE_ONCE(rb->aux_in_sampling, 1);
	barrier();

	ret = event->pmu->snapshot_aux(event, handle, size);

	barrier();
	WRITE_ONCE(rb->aux_in_sampling, 0);
	local_irq_restore(flags);

	return ret;
}


static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
{
	perf_event_cgroup(css->cgroup);
	return 0;
}


static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
{
	struct perf_event *event = vmf->vma->vm_file->private_data;
	struct perf_buffer *rb;
	vm_fault_t ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	rb = rcu_dereference(event->rb);
	if (!rb)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vmf->vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}


static void perf_group_attach(struct perf_event *event)
{
	struct perf_event *group_leader = event->group_leader, *pos;

	lockdep_assert_held(&event->ctx->lock);

	/*
	 * We can have double attach due to group movement in perf_event_open.
	 */
	if (event->attach_state & PERF_ATTACH_GROUP)
		return;

	event->attach_state |= PERF_ATTACH_GROUP;

	if (group_leader == event)
		return;

	WARN_ON_ONCE(group_leader->ctx != event->ctx);

	group_leader->group_caps &= event->event_caps;

	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
	group_leader->nr_siblings++;

	perf_event__header_size(group_leader);

	for_each_sibling_event(pos, group_leader)
		perf_event__header_size(pos);
}


void perf_event_disable_inatomic(struct perf_event *event)
{
	WRITE_ONCE(event->pending_disable, smp_processor_id());
	/* can fail, see perf_pending_event_disable() */
	irq_work_queue(&event->pending);
}


static int perf_event_text_poke_match(struct perf_event *event)
{
	return event->attr.text_poke;
}


static struct pmu *perf_init_event(struct perf_event *event)
{
	int idx, type, ret;
	struct pmu *pmu;

	idx = srcu_read_lock(&pmus_srcu);

	/* Try parent's PMU first: */
	if (event->parent && event->parent->pmu) {
		pmu = event->parent->pmu;
		ret = perf_try_init_event(pmu, event);
		if (!ret)
			goto unlock;
	}

	/*
	 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
	 * are often aliases for PERF_TYPE_RAW.
	 */
	type = event->attr.type;
	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE)
		type = PERF_TYPE_RAW;

again:
	rcu_read_lock();
	pmu = idr_find(&pmu_idr, type);
	rcu_read_unlock();
	if (pmu) {
		ret = perf_try_init_event(pmu, event);
		if (ret == -ENOENT && event->attr.type != type) {
			type = event->attr.type;
			goto again;
		}

		if (ret)
			pmu = ERR_PTR(ret);

		goto unlock;
	}

	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
		ret = perf_try_init_event(pmu, event);
		if (!ret)
			goto unlock;

		if (ret != -ENOENT) {
			pmu = ERR_PTR(ret);
			goto unlock;
		}
	}
	pmu = ERR_PTR(-ENOENT);
unlock:
	srcu_read_unlock(&pmus_srcu, idx);

	return pmu;
}


static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
{
	struct perf_event *child;
	u64 total = 0;

	*enabled = 0;
	*running = 0;

	mutex_lock(&event->child_mutex);

	(void)perf_event_read(event, false);
	total += perf_event_count(event);

	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
		(void)perf_event_read(child, false);
		total += perf_event_count(child);
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
	mutex_unlock(&event->child_mutex);

	return total;
}


struct file *perf_event_get(unsigned int fd)
{
	struct file *file = fget(fd);
	if (!file)
		return ERR_PTR(-EBADF);

	if (file->f_op != &perf_fops) {
		fput(file);
		return ERR_PTR(-EBADF);
	}

	return file;
}


static void perf_pending_event(struct irq_work *entry)
{
	struct perf_event *event = container_of(entry, struct perf_event, pending);
	int rctx;

	rctx = perf_swevent_get_recursion_context();
	/*
	 * If we 'fail' here, that's OK, it means recursion is already disabled
	 * and we won't recurse 'further'.
	 */

	perf_pending_event_disable(event);

	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
	}

	if (rctx >= 0)
		perf_swevent_put_recursion_context(rctx);
}


perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
			int cpu)
{
	struct task_struct *task = READ_ONCE(ctx->task);

	lockdep_assert_held(&ctx->mutex);

	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));

	if (event->cpu != -1)
		event->cpu = cpu;

	/*
	 * Ensures that if we can observe event->ctx, both the event and ctx
	 * will be 'complete'. See perf_iterate_sb_cpu().
	 */
	smp_store_release(&event->ctx, ctx);

	/*
	 * perf_event_attr::disabled events will not run and can be initialized
	 * without IPI. Except when this is the first event for the context, in
	 * that case we need the magic of the IPI to set ctx->is_active.
	 *
	 * The IOC_ENABLE that is sure to follow the creation of a disabled
	 * event will issue the IPI and reprogram the hardware.
	 */
	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && ctx->nr_events) {
		raw_spin_lock_irq(&ctx->lock);
		if (ctx->task == TASK_TOMBSTONE) {
			raw_spin_unlock_irq(&ctx->lock);
			return;
		}
		add_event_to_ctx(event, ctx);
		raw_spin_unlock_irq(&ctx->lock);
		return;
	}

	if (!task) {
		cpu_function_call(cpu, __perf_install_in_context, event);
		return;
	}

	/*
	 * Should not happen, we validate the ctx is still alive before calling.
	 */
	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
		return;

	/*
	 * Installing events is tricky because we cannot rely on ctx->is_active
	 * to be set in case this is the nr_events 0 -> 1 transition.
	 *
	 * Instead we use task_curr(), which tells us if the task is running.
	 * However, since we use task_curr() outside of rq::lock, we can race
	 * against the actual state. This means the result can be wrong.
	 *
	 * If we get a false positive, we retry, this is harmless.
	 *
	 * If we get a false negative, things are complicated. If we are after
	 * perf_event_context_sched_in() ctx::lock will serialize us, and the
	 * value must be correct. If we're before, it doesn't matter since
	 * perf_event_context_sched_in() will program the counter.
	 *
	 * However, this hinges on the remote context switch having observed
	 * our task->perf_event_ctxp[] store, such that it will in fact take
	 * ctx::lock in perf_event_context_sched_in().
	 *
	 * We do this by task_function_call(), if the IPI fails to hit the task
	 * we know any future context switch of task must see the
	 * perf_event_ctpx[] store.
	 */

	/*
	 * This smp_mb() orders the task->perf_event_ctxp[] store with the
	 * task_cpu() load, such that if the IPI then does not find the task
	 * running, a future context switch of that task must observe the
	 * store.
	 */
	smp_mb();
again:
	if (!task_function_call(task, __perf_install_in_context, event))
		return;

	raw_spin_lock_irq(&ctx->lock);
	task = ctx->task;
	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
		/*
		 * Cannot happen because we already checked above (which also
		 * cannot happen), and we hold ctx->mutex, which serializes us
		 * against perf_event_exit_task_context().
		 */
		raw_spin_unlock_irq(&ctx->lock);
		return;
	}
	/*
	 * If the task is not running, ctx->lock will avoid it becoming so,
	 * thus we can safely install the event.
	 */
	if (task_curr(task)) {
		raw_spin_unlock_irq(&ctx->lock);
		goto again;
	}
	add_event_to_ctx(event, ctx);
	raw_spin_unlock_irq(&ctx->lock);
}


int perf_event_period(struct perf_event *event, u64 value)
{
	struct perf_event_context *ctx;
	int ret;

	ctx = perf_event_ctx_lock(event);
	ret = _perf_event_period(event, value);
	perf_event_ctx_unlock(event, ctx);

	return ret;
}


static void perf_event_text_poke_output(struct perf_event *event, void *data)
{
	struct perf_text_poke_event *text_poke_event = data;
	struct perf_output_handle handle;
	struct perf_sample_data sample;
	u64 padding = 0;
	int ret;

	if (!perf_event_text_poke_match(event))
		return;

	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);

	ret = perf_output_begin(&handle, event, text_poke_event->event_id.header.size);
	if (ret)
		return;

	perf_output_put(&handle, text_poke_event->event_id);
	perf_output_put(&handle, text_poke_event->old_len);
	perf_output_put(&handle, text_poke_event->new_len);

	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);

	if (text_poke_event->pad)
		__output_copy(&handle, &padding, text_poke_event->pad);

	perf_event__output_id_sample(event, &handle, &sample);

	perf_output_end(&handle);
}


void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
		   struct pt_regs *regs, struct hlist_head *head, int rctx,
		   struct task_struct *task)
{
	struct perf_sample_data data;
	struct perf_event *event;

	struct perf_raw_record raw = {
		.frag = {
			.size = entry_size,
			.data = record,
		},
	};

	perf_sample_data_init(&data, 0, 0);
	data.raw = &raw;

	perf_trace_buf_update(record, event_type);

	hlist_for_each_entry_rcu(event, head, hlist_entry) {
		if (perf_tp_event_match(event, &data, regs))
			perf_swevent_event(event, count, &data, regs);
	}

	/*
	 * If we got specified a target task, also iterate its context and
	 * deliver this event there too.
	 */
	if (task && task != current) {
		struct perf_event_context *ctx;
		struct trace_entry *entry = record;

		rcu_read_lock();
		ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
		if (!ctx)
			goto unlock;

		list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
			if (event->cpu != smp_processor_id())
				continue;
			if (event->attr.type != PERF_TYPE_TRACEPOINT)
				continue;
			if (event->attr.config != entry->type)
				continue;
			if (perf_tp_event_match(event, &data, regs))
				perf_swevent_event(event, count, &data, regs);
		}
unlock:
		rcu_read_unlock();
	}

	perf_swevent_put_recursion_context(rctx);
}


static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);

	return 1;
}


void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
			struct perf_event *event)
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IDENTIFIER)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
		perf_output_read(handle, event);

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		int size = 1;

		size += data->callchain->nr;
		size *= sizeof(u64);
		__output_copy(handle, data->callchain, size);
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		struct perf_raw_record *raw = data->raw;

		if (raw) {
			struct perf_raw_frag *frag = &raw->frag;

			perf_output_put(handle, raw->size);
			do {
				if (frag->copy) {
					__output_custom(handle, frag->copy,
							frag->data, frag->size);
				} else {
					__output_copy(handle, frag->data,
						      frag->size);
				}
				if (perf_raw_frag_last(frag))
					break;
				frag = frag->next;
			} while (1);
			if (frag->pad)
				__output_skip(handle, NULL, frag->pad);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}

	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
		if (data->br_stack) {
			size_t size;

			size = data->br_stack->nr
			     * sizeof(struct perf_branch_entry);

			perf_output_put(handle, data->br_stack->nr);
			if (perf_sample_save_hw_index(event))
				perf_output_put(handle, data->br_stack->hw_idx);
			perf_output_copy(handle, data->br_stack->entries, size);
		} else {
			/*
			 * we always store at least the value of nr
			 */
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_REGS_USER) {
		u64 abi = data->regs_user.abi;

		/*
		 * If there are no regs to dump, notice it through
		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
		 */
		perf_output_put(handle, abi);

		if (abi) {
			u64 mask = event->attr.sample_regs_user;
			perf_output_sample_regs(handle,
						data->regs_user.regs,
						mask);
		}
	}

	if (sample_type & PERF_SAMPLE_STACK_USER) {
		perf_output_sample_ustack(handle,
					  data->stack_user_size,
					  data->regs_user.regs);
	}

	if (sample_type & PERF_SAMPLE_WEIGHT)
		perf_output_put(handle, data->weight);

	if (sample_type & PERF_SAMPLE_DATA_SRC)
		perf_output_put(handle, data->data_src.val);

	if (sample_type & PERF_SAMPLE_TRANSACTION)
		perf_output_put(handle, data->txn);

	if (sample_type & PERF_SAMPLE_REGS_INTR) {
		u64 abi = data->regs_intr.abi;
		/*
		 * If there are no regs to dump, notice it through
		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
		 */
		perf_output_put(handle, abi);

		if (abi) {
			u64 mask = event->attr.sample_regs_intr;

			perf_output_sample_regs(handle,
						data->regs_intr.regs,
						mask);
		}
	}

	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
		perf_output_put(handle, data->phys_addr);

	if (sample_type & PERF_SAMPLE_CGROUP)
		perf_output_put(handle, data->cgroup);

	if (sample_type & PERF_SAMPLE_AUX) {
		perf_output_put(handle, data->aux_size);

		if (data->aux_size)
			perf_aux_sample_output(event, handle, data);
	}

	if (!event->attr.watermark) {
		int wakeup_events = event->attr.wakeup_events;

		if (wakeup_events) {
			struct perf_buffer *rb = handle->rb;
			int events = local_inc_return(&rb->events);

			if (events >= wakeup_events) {
				local_sub(wakeup_events, &rb->events);
				local_inc(&rb->wakeup);
			}
		}
	}
}


perf_event_groups_delete(struct perf_event_groups *groups,
			 struct perf_event *event)
{
	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
		     RB_EMPTY_ROOT(&groups->tree));

	rb_erase(&event->group_node, &groups->tree);
	init_event_group(event);
}


static inline void perf_remove_sibling_event(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	event_sched_out(event, cpuctx, ctx);
	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
}


perf_event_groups_less(struct perf_event *left, struct perf_event *right)
{
	if (left->cpu < right->cpu)
		return true;
	if (left->cpu > right->cpu)
		return false;

#ifdef CONFIG_CGROUP_PERF
	if (left->cgrp != right->cgrp) {
		if (!left->cgrp || !left->cgrp->css.cgroup) {
			/*
			 * Left has no cgroup but right does, no cgroups come
			 * first.
			 */
			return true;
		}
		if (!right->cgrp || !right->cgrp->css.cgroup) {
			/*
			 * Right has no cgroup but left does, no cgroups come
			 * first.
			 */
			return false;
		}
		/* Two dissimilar cgroups, order by id. */
		if (left->cgrp->css.cgroup->kn->id < right->cgrp->css.cgroup->kn->id)
			return true;

		return false;
	}
#endif

	if (left->group_index < right->group_index)
		return true;
	if (left->group_index > right->group_index)
		return false;

	return false;
}


static void perf_event_bpf_output(struct perf_event *event, void *data)
{
	struct perf_bpf_event *bpf_event = data;
	struct perf_output_handle handle;
	struct perf_sample_data sample;
	int ret;

	if (!perf_event_bpf_match(event))
		return;

	perf_event_header__init_id(&bpf_event->event_id.header,
				   &sample, event);
	ret = perf_output_begin(&handle, event,
				bpf_event->event_id.header.size);
	if (ret)
		return;

	perf_output_put(&handle, bpf_event->event_id);
	perf_event__output_id_sample(event, &handle, &sample);

	perf_output_end(&handle);
}


void perf_sched_cb_dec(struct pmu *pmu)
{
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

	--cpuctx->sched_cb_usage;
}


raptor_xml_writer_raw_counted(raptor_xml_writer* xml_writer,
                              const unsigned char *s, unsigned int len)
{
  raptor_xml_writer_write_xml_declaration(xml_writer);

  XML_WRITER_FLUSH_CLOSE_BRACKET(xml_writer);

  raptor_iostream_counted_string_write(s, len, xml_writer->iostr);

  if(xml_writer->current_element)
    xml_writer->current_element->content_cdata_seen = 1;
}


raptor_xml_writer_comment(raptor_xml_writer* xml_writer,
                          const unsigned char *s)
{
  XML_WRITER_FLUSH_CLOSE_BRACKET(xml_writer);

  raptor_xml_writer_raw_counted(xml_writer, (const unsigned char*)"<!-- ", 5);
  raptor_xml_writer_cdata(xml_writer, s);
  raptor_xml_writer_raw_counted(xml_writer, (const unsigned char*)" -->", 4);
}


pmixp_coll_ring_ctx_t *pmixp_coll_ring_ctx_select(pmixp_coll_t *coll,
						  const uint32_t seq)
{
	int i;
	pmixp_coll_ring_ctx_t *coll_ctx = NULL, *ret = NULL;
	pmixp_coll_ring_t *ring = &coll->state.ring;

	/* finding the appropriate ring context */
	for (i = 0; i < PMIXP_COLL_RING_CTX_NUM; i++) {
		coll_ctx = &ring->ctx_array[i];
		if (coll_ctx->in_use && coll_ctx->seq == seq) {
			return coll_ctx;
		} else if (!coll_ctx->in_use) {
			ret = coll_ctx;
			continue;
		}
	}
	/* add this context to use */
	if (ret && !ret->in_use) {
		ret->in_use = true;
		ret->seq = seq;
		ret->ring_buf = _get_contrib_buf(ret);
	}
	return ret;
}


int pmixp_coll_tree_child(pmixp_coll_t *coll, uint32_t peerid,
			  uint32_t seq, Buf buf)
{
	char *data_src = NULL, *data_dst = NULL;
	uint32_t size;
	int chld_id;
	pmixp_coll_tree_t *tree = NULL;

	/* lock the structure */
	slurm_mutex_lock(&coll->lock);
	pmixp_coll_sanity_check(coll);
	tree = &coll->state.tree;

	if (0 > (chld_id = _chld_id(tree, peerid))) {
		char *nodename = pmixp_info_job_host(peerid);
		char *avail_ids = _chld_ids_str(tree);
		PMIXP_DEBUG("%p: contribution from the non-child node %s:%u, acceptable ids: %s",
			    coll, nodename, peerid, avail_ids);
		xfree(nodename);
		xfree(avail_ids);
	}

#ifdef PMIXP_COLL_DEBUG
	PMIXP_DEBUG("%p: contrib/rem from nodeid=%u, childid=%d, state=%s, size=%u",
		    coll, peerid, chld_id,
		    pmixp_coll_tree_state2str(tree->state),
		    remaining_buf(buf));
#endif

	switch (tree->state) {
	case PMIXP_COLL_TREE_SYNC:
		/* change the state */
		coll->ts = time(NULL);
		/* fall-thru */
	case PMIXP_COLL_TREE_COLLECT:
		/* sanity check */
		if (coll->seq != seq) {
			char *nodename = pmixp_info_job_host(peerid);
			/* FATAL: should not happen in normal workflow */
			PMIXP_ERROR("%p: unexpected contrib from %s:%d (child #%d) seq = %d, coll->seq = %d, state=%s",
				    coll, nodename, peerid, chld_id,
				    seq, coll->seq,
				    pmixp_coll_tree_state2str(tree->state));
			xfree(nodename);
			goto error;
		}
		break;
	case PMIXP_COLL_TREE_UPFWD:
	case PMIXP_COLL_TREE_UPFWD_WSC:
	{
		char *nodename = pmixp_info_job_host(peerid);
		/* FATAL: should not happen in normal workflow */
		PMIXP_ERROR("%p: unexpected contrib from %s:%d, state = %s",
			    coll, nodename, peerid,
			    pmixp_coll_tree_state2str(tree->state));
		xfree(nodename);
		goto error;
	}
	case PMIXP_COLL_TREE_UPFWD_WPC:
	case PMIXP_COLL_TREE_DOWNFWD:
#ifdef PMIXP_COLL_DEBUG
		/* It looks like a retransmission attempt when remote side
		 * identified transmission failure, but we actually successfuly
		 * received the message */
		PMIXP_DEBUG("%p: contrib for the next coll. nodeid=%u, child=%d seq=%u, coll->seq=%u, state=%s",
			    coll, peerid, chld_id, seq, coll->seq,
			    pmixp_coll_tree_state2str(tree->state));
#endif
		if ((coll->seq +1) != seq) {
			char *nodename = pmixp_info_job_host(peerid);
			/* should not happen in normal workflow */
			PMIXP_ERROR("%p: unexpected contrib from %s:%d(x:%d) seq = %d, coll->seq = %d, state=%s",
				    coll, nodename, peerid, chld_id,
				    seq, coll->seq,
				    pmixp_coll_tree_state2str(tree->state));
			xfree(nodename);
			goto error;
		}
		break;
	default:
		/* should not happen in normal workflow */
		PMIXP_ERROR("%p: unknown collective state %s",
			    coll, pmixp_coll_tree_state2str(tree->state));
		/* collective is spoiled, reset state */
		tree->state = PMIXP_COLL_TREE_SYNC;
		goto error2;
	}

	/* Because of possible timeouts/delays in transmission we
	 * can receive a contribution multiple times. Avoid duplications
	 * by checking our records. */
	if (tree->contrib_chld[chld_id]) {
		char *nodename = pmixp_info_job_host(peerid);
		/* May be 0 or 1. If grater - transmission skew, ignore.
		 * NOTE: this output is not on the critical path -
		 * don't preprocess it out */
		PMIXP_DEBUG("%p: multiple contribs from %s:%d(x:%d)",
			    coll, nodename, peerid, chld_id);
		/* this is duplication, skip. */
		xfree(nodename);
		goto proceed;
	}

	data_src = get_buf_data(buf) + get_buf_offset(buf);
	size = remaining_buf(buf);
	pmixp_server_buf_reserve(tree->ufwd_buf, size);
	data_dst = get_buf_data(tree->ufwd_buf) +
		get_buf_offset(tree->ufwd_buf);
	memcpy(data_dst, data_src, size);
	set_buf_offset(tree->ufwd_buf, get_buf_offset(tree->ufwd_buf) + size);

	/* increase number of individual contributions */
	tree->contrib_chld[chld_id] = true;
	/* increase number of total contributions */
	tree->contrib_children++;

proceed:
	_progress_coll_tree(coll);

#ifdef PMIXP_COLL_DEBUG
	PMIXP_DEBUG("%p: finish nodeid=%u, child=%d, state=%s",
		    coll, peerid, chld_id,
		    pmixp_coll_tree_state2str(tree->state));
#endif
	/* unlock the structure */
	slurm_mutex_unlock(&coll->lock);

	return SLURM_SUCCESS;
error:
	pmixp_coll_log(coll);
	_reset_coll(coll);
error2:
	slurm_kill_job_step(pmixp_info_jobid(),
			    pmixp_info_stepid(), SIGKILL);
	/* unlock the structure */
	slurm_mutex_unlock(&coll->lock);

	return SLURM_ERROR;
}


static int _pack_coll_ring_info(pmixp_coll_t *coll,
				pmixp_coll_ring_msg_hdr_t *ring_hdr,
				Buf buf)
{
	pmixp_proc_t *procs = coll->pset.procs;
	size_t nprocs = coll->pset.nprocs;
	uint32_t type = PMIXP_COLL_TYPE_FENCE_RING;
	int i;

	/* 1. store the type of collective */
	pack32(type, buf);

	/* 2. Put the number of ranges */
	pack32(nprocs, buf);
	for (i = 0; i < (int)nprocs; i++) {
		/* Pack namespace */
		packmem(procs->nspace, strlen(procs->nspace) + 1, buf);
		pack32(procs->rank, buf);
	}

	/* 3. pack the ring header info */
	packmem((char*)ring_hdr, sizeof(pmixp_coll_ring_msg_hdr_t), buf);

	return SLURM_SUCCESS;
}


static int _chld_id(pmixp_coll_tree_t *tree, uint32_t nodeid)
{
	int i;

	for (i=0; i<tree->chldrn_cnt; i++) {
		if (tree->chldrn_ids[i] == nodeid) {
			return i;
		}
	}
	return -1;
}


static inline pmixp_coll_t *_ctx_get_coll(pmixp_coll_ring_ctx_t *coll_ctx)
{
	return coll_ctx->coll;
}


int pmixp_coll_ring_check(pmixp_coll_t *coll, pmixp_coll_ring_msg_hdr_t *hdr)
{
	char *nodename = NULL;
	int rc;

	if (hdr->nodeid != _ring_prev_id(coll)) {
		nodename = pmixp_info_job_host(hdr->nodeid);
		PMIXP_ERROR("%p: unexpected contrib from %s:%u, expected is %d",
			    coll, nodename, hdr->nodeid, _ring_prev_id(coll));
		return SLURM_ERROR;
	}
	rc = pmixp_coll_check(coll, hdr->seq);
	if (PMIXP_COLL_REQ_FAILURE == rc) {
		/* this is an unacceptable event: either something went
		 * really wrong or the state machine is incorrect.
		 * This will 100% lead to application hang.
		 */
		nodename = pmixp_info_job_host(hdr->nodeid);
		PMIXP_ERROR("Bad collective seq. #%d from %s:%u, current is %d",
			    hdr->seq, nodename, hdr->nodeid, coll->seq);
		pmixp_debug_hang(0); /* enable hang to debug this! */
		slurm_kill_job_step(pmixp_info_jobid(),
				    pmixp_info_stepid(), SIGKILL);
		xfree(nodename);
		return SLURM_SUCCESS;
	} else if (PMIXP_COLL_REQ_SKIP == rc) {
#ifdef PMIXP_COLL_DEBUG
		nodename = pmixp_info_job_host(hdr->nodeid);
		PMIXP_ERROR("Wrong collective seq. #%d from nodeid %u, current is %d, skip this message",
			    hdr->seq, hdr->nodeid, coll->seq);
#endif
		return SLURM_ERROR;
	}
	return SLURM_SUCCESS;
}


static int _progress_collect(pmixp_coll_t *coll)
{
	pmixp_ep_t ep = {0};
	int rc;
	pmixp_coll_tree_t *tree = &coll->state.tree;

	xassert(PMIXP_COLL_TREE_COLLECT == tree->state);

	ep.type = PMIXP_EP_NONE;
#ifdef PMIXP_COLL_DEBUG
	PMIXP_DEBUG("%p: state=%s, local=%d, child_cntr=%d",
		    coll, pmixp_coll_tree_state2str(tree->state),
		    (int)tree->contrib_local, tree->contrib_children);
#endif
	/* sanity check */
	pmixp_coll_sanity_check(coll);

	if (PMIXP_COLL_TREE_COLLECT != tree->state) {
		/* In case of race condition between libpmix and
		 * slurm threads we can be called
		 * after we moved to the next step. */
		return 0;
	}

	if (!tree->contrib_local ||
	    tree->contrib_children != tree->chldrn_cnt) {
		/* Not yet ready to go to the next step */
		return 0;
	}

	if (pmixp_info_srv_direct_conn()) {
		/* We will need to forward aggregated
		 * message back to our children */
		tree->state = PMIXP_COLL_TREE_UPFWD;
	} else {
		/* If we use Slurm API (SAPI) - intermediate nodes
		 * don't need to forward data as the root will do
		 * SAPI broadcast.
		 * So, only root has to go through the full UPFWD
		 * state and send the message back.
		 * Other procs have to go through other route. The reason for
		 * that is the fact that som of out children can receive bcast
		 * message early and initiate next collective. We need to handle
		 * that properly.
		 */
		if (0 > tree->prnt_peerid) {
			tree->state = PMIXP_COLL_TREE_UPFWD;
		} else {
			tree->state = PMIXP_COLL_TREE_UPFWD_WSC;
		}
	}

	/* The root of the collective will have parent_host == NULL */
	if (NULL != tree->prnt_host) {
		ep.type = PMIXP_EP_NOIDEID;
		ep.ep.nodeid = tree->prnt_peerid;
		tree->ufwd_status = PMIXP_COLL_TREE_SND_ACTIVE;
		PMIXP_DEBUG("%p: send data to %s:%d",
			    coll, tree->prnt_host, tree->prnt_peerid);
	} else {
		/* move data from input buffer to the output */
		char *dst, *src = get_buf_data(tree->ufwd_buf) +
			tree->ufwd_offset;
		size_t size = get_buf_offset(tree->ufwd_buf) -
			tree->ufwd_offset;
		pmixp_server_buf_reserve(tree->dfwd_buf, size);
		dst = get_buf_data(tree->dfwd_buf) + tree->dfwd_offset;
		memcpy(dst, src, size);
		set_buf_offset(tree->dfwd_buf, tree->dfwd_offset + size);
		/* no need to send */
		tree->ufwd_status = PMIXP_COLL_TREE_SND_DONE;
		/* this is root */
		tree->contrib_prnt = true;
	}

	if (PMIXP_EP_NONE != ep.type) {
		pmixp_coll_cbdata_t *cbdata;
		cbdata = xmalloc(sizeof(pmixp_coll_cbdata_t));
		cbdata->coll = coll;
		cbdata->seq = coll->seq;
		cbdata->refcntr = 1;
		char *nodename = tree->prnt_host;
		rc = pmixp_server_send_nb(&ep, PMIXP_MSG_FAN_IN, coll->seq,
					  tree->ufwd_buf,
					  _ufwd_sent_cb, cbdata);

		if (SLURM_SUCCESS != rc) {
			PMIXP_ERROR("Cannot send data (size = %u), to %s:%d",
				    get_buf_offset(tree->ufwd_buf),
				    nodename, ep.ep.nodeid);
			tree->ufwd_status = PMIXP_COLL_TREE_SND_FAILED;
		}
#ifdef PMIXP_COLL_DEBUG
		PMIXP_DEBUG("%p: fwd to %s:%d, size = %u",
			    coll, nodename, ep.ep.nodeid,
			    get_buf_offset(tree->dfwd_buf));
#endif
	}

	/* events observed - need another iteration */
	return true;
}


void pmixp_coll_ring_free(pmixp_coll_ring_t *ring)
{
	int i;

	pmixp_coll_ring_ctx_t *coll_ctx;
	for (i = 0; i < PMIXP_COLL_RING_CTX_NUM; i++) {
		coll_ctx = &ring->ctx_array[i];
		FREE_NULL_BUFFER(coll_ctx->ring_buf);
		xfree(coll_ctx->contrib_map);
	}
	list_destroy(ring->fwrd_buf_pool);
	list_destroy(ring->ring_buf_pool);
}


static inline uint32_t _ring_remain_contrib(pmixp_coll_ring_ctx_t *coll_ctx)
{
	return coll_ctx->coll->peers_cnt -
		(coll_ctx->contrib_prev + coll_ctx->contrib_local);
}


_cairo_image_finish_unbounded_spans (void *abstract_renderer)
{
    cairo_image_span_renderer_t *r = abstract_renderer;

    if (r->extents.y < r->extents.height) {
	pixman_image_compositor_blt (r->compositor,
				     r->extents.x, r->extents.y,
				     r->extents.width,
				     r->extents.height - r->extents.y,
				     0);
    }

    return CAIRO_STATUS_SUCCESS;
}


draw_image_boxes (void *_dst,
		  cairo_image_surface_t *image,
		  cairo_boxes_t *boxes,
		  int dx, int dy)
{
    cairo_image_surface_t *dst = _dst;
    struct _cairo_boxes_chunk *chunk;
    int i;

    TRACE ((stderr, "%s x %d\n", __FUNCTION__, boxes->num_boxes));

    for (chunk = &boxes->chunks; chunk; chunk = chunk->next) {
	for (i = 0; i < chunk->count; i++) {
	    cairo_box_t *b = &chunk->base[i];
	    int x = _cairo_fixed_integer_part (b->p1.x);
	    int y = _cairo_fixed_integer_part (b->p1.y);
	    int w = _cairo_fixed_integer_part (b->p2.x) - x;
	    int h = _cairo_fixed_integer_part (b->p2.y) - y;
	    if (dst->pixman_format != image->pixman_format ||
		! pixman_blt ((uint32_t *)image->data, (uint32_t *)dst->data,
			      image->stride / sizeof (uint32_t),
			      dst->stride / sizeof (uint32_t),
			      PIXMAN_FORMAT_BPP (image->pixman_format),
			      PIXMAN_FORMAT_BPP (dst->pixman_format),
			      x + dx, y + dy,
			      x, y,
			      w, h))
	    {
		pixman_image_composite32 (PIXMAN_OP_SRC,
					  image->pixman_image, NULL, dst->pixman_image,
					  x + dx, y + dy,
					  0, 0,
					  x, y,
					  w, h);
	    }
	}
    }
    return CAIRO_STATUS_SUCCESS;
}


to_pixman_image (cairo_surface_t *s)
{
    return ((cairo_image_surface_t *)s)->pixman_image;
}


static void free_pixels (pixman_image_t *image, void *data)
{
	free (data);
}


_mono_unbounded_spans (void *abstract_renderer, int y, int h,
		       const cairo_half_open_span_t *spans, unsigned num_spans)
{
    cairo_image_span_renderer_t *r = abstract_renderer;

    if (num_spans == 0) {
	pixman_image_composite32 (PIXMAN_OP_CLEAR,
				  r->src, NULL, r->u.composite.dst,
				  spans[0].x + r->u.composite.src_x,  y + r->u.composite.src_y,
				  0, 0,
				  r->composite->unbounded.x, y,
				  r->composite->unbounded.width, h);
	r->u.composite.mask_y = y + h;
	return CAIRO_STATUS_SUCCESS;
    }

    if (y != r->u.composite.mask_y) {
	pixman_image_composite32 (PIXMAN_OP_CLEAR,
				  r->src, NULL, r->u.composite.dst,
				  spans[0].x + r->u.composite.src_x,  y + r->u.composite.src_y,
				  0, 0,
				  r->composite->unbounded.x, r->u.composite.mask_y,
				  r->composite->unbounded.width, y - r->u.composite.mask_y);
    }

    if (spans[0].x != r->composite->unbounded.x) {
	    pixman_image_composite32 (PIXMAN_OP_CLEAR,
				      r->src, NULL, r->u.composite.dst,
				      spans[0].x + r->u.composite.src_x,  y + r->u.composite.src_y,
				      0, 0,
				      r->composite->unbounded.x, y,
				      spans[0].x - r->composite->unbounded.x, h);
    }

    do {
	int op = spans[0].coverage ? r->op : PIXMAN_OP_CLEAR;
	pixman_image_composite32 (op,
				  r->src, NULL, r->u.composite.dst,
				  spans[0].x + r->u.composite.src_x,  y + r->u.composite.src_y,
				  0, 0,
				  spans[0].x, y,
				  spans[1].x - spans[0].x, h);
	spans++;
    } while (--num_spans > 1);

    if (spans[0].x != r->composite->unbounded.x + r->composite->unbounded.width) {
	    pixman_image_composite32 (PIXMAN_OP_CLEAR,
				      r->src, NULL, r->u.composite.dst,
				      spans[0].x + r->u.composite.src_x,  y + r->u.composite.src_y,
				      0, 0,
				      spans[0].x, y,
				      r->composite->unbounded.x + r->composite->unbounded.width - spans[0].x, h);
    }

    r->u.composite.mask_y = y + h;
    return CAIRO_STATUS_SUCCESS;
}


span_renderer_fini (cairo_abstract_span_renderer_t *_r,
		    cairo_int_status_t status)
{
    cairo_image_span_renderer_t *r = (cairo_image_span_renderer_t *) _r;

    TRACE ((stderr, "%s\n", __FUNCTION__));

    if (status == CAIRO_INT_STATUS_SUCCESS && r->base.finish)
	r->base.finish (r);

    if (r->compositor)
	pixman_image_compositor_destroy (r->compositor);

    if (r->src)
	pixman_image_unref (r->src);
    if (r->mask)
	pixman_image_unref (r->mask);
}


csnNormalize(
	slap_mask_t usage,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *val,
	struct berval *normalized,
	void *ctx )
{
	struct berval	cnt, sid, mod;
	char		*ptr;
	ber_len_t	i;

	assert( val != NULL );
	assert( normalized != NULL );

	assert( SLAP_MR_IS_VALUE_OF_SYNTAX( usage ) != 0 );

	if ( BER_BVISEMPTY( val ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	if ( val->bv_len == STRLENOF( "YYYYmmddHHMMSSZ#SSSSSS#ID#ssssss" ) ) {
		/* Openldap <= 2.3 */

		return csnNormalize23( usage, syntax, mr, val, normalized, ctx );
	}

	if ( val->bv_len == STRLENOF( "YYYYmmddHH:MM:SSZ#0xSSSS#I#ssss" ) ) {
		/* Openldap 2.1 */

		return csnNormalize21( usage, syntax, mr, val, normalized, ctx );
	}

	if ( val->bv_len != STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ#SSSSSS#SID#ssssss" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	ptr = ber_bvchr( val, '#' );
	if ( ptr == NULL || ptr == &val->bv_val[val->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	if ( ptr - val->bv_val != STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	cnt.bv_val = ptr + 1;
	cnt.bv_len = val->bv_len - ( cnt.bv_val - val->bv_val );

	ptr = ber_bvchr( &cnt, '#' );
	if ( ptr == NULL || ptr == &val->bv_val[val->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	if ( ptr - cnt.bv_val != STRLENOF( "000000" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	sid.bv_val = ptr + 1;
	sid.bv_len = val->bv_len - ( sid.bv_val - val->bv_val );

	ptr = ber_bvchr( &sid, '#' );
	if ( ptr == NULL || ptr == &val->bv_val[val->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	sid.bv_len = ptr - sid.bv_val;
	if ( sid.bv_len != STRLENOF( "000" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	mod.bv_val = ptr + 1;
	mod.bv_len = val->bv_len - ( mod.bv_val - val->bv_val );

	if ( mod.bv_len != STRLENOF( "000000" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	ber_dupbv_x( normalized, val, ctx );

	for ( i = STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ#SSSSSS#" );
		i < normalized->bv_len; i++ )
	{
		/* assume it's already validated that's all hex digits */
		normalized->bv_val[ i ] = TOLOWER( normalized->bv_val[ i ] );
	}

	return LDAP_SUCCESS;
}


countryStringValidate(
	Syntax *syntax,
	struct berval *val )
{
	if( val->bv_len != 2 ) return LDAP_INVALID_SYNTAX;

	if( !SLAP_PRINTABLE(val->bv_val[0]) ) {
		return LDAP_INVALID_SYNTAX;
	}
	if( !SLAP_PRINTABLE(val->bv_val[1]) ) {
		return LDAP_INVALID_SYNTAX;
	}

	return LDAP_SUCCESS;
}


firstComponentNormalize(
	slap_mask_t usage,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *val,
	struct berval *normalized,
	void *ctx )
{
	int rc;
	struct berval comp;
	ber_len_t len;

	if( SLAP_MR_IS_VALUE_OF_ASSERTION_SYNTAX( usage )) {
		ber_dupbv_x( normalized, val, ctx );
		return LDAP_SUCCESS;
	}

	if( val->bv_len < 3 ) return LDAP_INVALID_SYNTAX;

	if( ! ( val->bv_val[0] == '(' /*')'*/
			&& val->bv_val[val->bv_len - 1] == /*'('*/ ')' )
		&& ! ( val->bv_val[0] == '{' /*'}'*/
			&& val->bv_val[val->bv_len - 1] == /*'('*/ '}' ) )
	{
		return LDAP_INVALID_SYNTAX;
	}

	/* trim leading white space */
	for( len=1;
		len < val->bv_len && ASCII_SPACE(val->bv_val[len]);
		len++ )
	{
		/* empty */
	}

	/* grab next word */
	comp.bv_val = &val->bv_val[len];
	len = val->bv_len - len - STRLENOF(/*"{"*/ "}");
	for( comp.bv_len = 0;
		!ASCII_SPACE(comp.bv_val[comp.bv_len]) && comp.bv_len < len;
		comp.bv_len++ )
	{
		/* empty */
	}

	if( mr == slap_schema.si_mr_objectIdentifierFirstComponentMatch ) {
		rc = numericoidValidate( NULL, &comp );
	} else if( mr == slap_schema.si_mr_integerFirstComponentMatch ) {
		rc = integerValidate( NULL, &comp );
	} else {
		rc = LDAP_INVALID_SYNTAX;
	}

	if( rc == LDAP_SUCCESS ) {
		ber_dupbv_x( normalized, &comp, ctx );
	}

	return rc;
}


csnValidate(
	Syntax *syntax,
	struct berval *in )
{
	struct berval	bv;
	char		*ptr;
	int		rc;

	assert( in != NULL );
	assert( !BER_BVISNULL( in ) );

	if ( BER_BVISEMPTY( in ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	bv = *in;

	ptr = ber_bvchr( &bv, '#' );
	if ( ptr == NULL || ptr == &bv.bv_val[bv.bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	bv.bv_len = ptr - bv.bv_val;
	if ( bv.bv_len != STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ" ) &&
		bv.bv_len != STRLENOF( "YYYYmmddHHMMSSZ" ) )
	{
		return LDAP_INVALID_SYNTAX;
	}

	rc = generalizedTimeValidate( NULL, &bv );
	if ( rc != LDAP_SUCCESS ) {
		return rc;
	}

	bv.bv_val = ptr + 1;
	bv.bv_len = in->bv_len - ( bv.bv_val - in->bv_val );

	ptr = ber_bvchr( &bv, '#' );
	if ( ptr == NULL || ptr == &in->bv_val[in->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	bv.bv_len = ptr - bv.bv_val;
	if ( bv.bv_len != 6 ) {
		return LDAP_INVALID_SYNTAX;
	}

	rc = hexValidate( NULL, &bv );
	if ( rc != LDAP_SUCCESS ) {
		return rc;
	}

	bv.bv_val = ptr + 1;
	bv.bv_len = in->bv_len - ( bv.bv_val - in->bv_val );

	ptr = ber_bvchr( &bv, '#' );
	if ( ptr == NULL || ptr == &in->bv_val[in->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	bv.bv_len = ptr - bv.bv_val;
	if ( bv.bv_len == 2 ) {
		/* tolerate old 2-digit replica-id */
		rc = hexValidate( NULL, &bv );

	} else {
		rc = sidValidate( NULL, &bv );
	}
	if ( rc != LDAP_SUCCESS ) {
		return rc;
	}

	bv.bv_val = ptr + 1;
	bv.bv_len = in->bv_len - ( bv.bv_val - in->bv_val );

	if ( bv.bv_len != 6 ) {
		return LDAP_INVALID_SYNTAX;
	}

	return hexValidate( NULL, &bv );
}


issuerAndThisUpdateNormalize(
	slap_mask_t usage,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *in,
	struct berval *out,
	void *ctx )
{
	struct berval i, ni, tu, tu2;
	char sbuf[STRLENOF("YYYYmmddHHMMSSZ") + 1];
	char *p;
	int rc;

	assert( in != NULL );
	assert( out != NULL );

	Debug( LDAP_DEBUG_TRACE, ">>> issuerAndThisUpdateNormalize: <%s>\n",
		in->bv_val, 0, 0 );

	rc = issuerAndThisUpdateCheck( in, &i, &tu, ctx );
	if ( rc ) {
		return rc;
	}

	rc = dnNormalize( usage, syntax, mr, &i, &ni, ctx );

	if ( in->bv_val[0] == '{' && in->bv_val[in->bv_len-1] == '}' ) {
		slap_sl_free( i.bv_val, ctx );
	}

	tu2.bv_val = sbuf;
	tu2.bv_len = sizeof( sbuf );
	if ( rc || checkTime( &tu, &tu2 ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	out->bv_len = STRLENOF( "{ issuer rdnSequence:\"\", thisUpdate \"\" }" )
		+ ni.bv_len + tu2.bv_len;
	out->bv_val = slap_sl_malloc( out->bv_len + 1, ctx );

	if ( out->bv_val == NULL ) {
		out->bv_len = 0;
		rc = LDAP_OTHER;
		goto func_leave;
	}

	p = out->bv_val;

	p = lutil_strcopy( p, "{ issuer rdnSequence:\"" /*}*/ );
	p = lutil_strbvcopy( p, &ni );
	p = lutil_strcopy( p, "\", thisUpdate \"" );
	p = lutil_strbvcopy( p, &tu2 );
	p = lutil_strcopy( p, /*{*/ "\" }" );

	assert( p == &out->bv_val[out->bv_len] );

func_leave:
	Debug( LDAP_DEBUG_TRACE, "<<< issuerAndThisUpdateNormalize: <%s> => <%s>\n",
		in->bv_val, rc == LDAP_SUCCESS ? out->bv_val : "(err)", 0 );

	slap_sl_free( ni.bv_val, ctx );

	return rc;
}


hashIter(
	HASH_CONTEXT *HASHcontext,
	unsigned char *HASHdigest,
	unsigned char *value,
	int len)
{
	HASH_CONTEXT ctx = *HASHcontext;
	HASH_Update( &ctx, value, len );
	HASH_Final( HASHdigest, &ctx );
}


csnNormalize21(
	slap_mask_t usage,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *val,
	struct berval *normalized,
	void *ctx )
{
	struct berval	gt, cnt, sid, mod;
	struct berval	bv;
	char		buf[ STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ#SSSSSS#SID#ssssss" ) + 1 ];
	char		*ptr;
	ber_len_t	i;

	assert( SLAP_MR_IS_VALUE_OF_SYNTAX( usage ) != 0 );
	assert( !BER_BVISEMPTY( val ) );

	gt = *val;

	ptr = ber_bvchr( &gt, '#' );
	if ( ptr == NULL || ptr == &gt.bv_val[gt.bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	gt.bv_len = ptr - gt.bv_val;
	if ( gt.bv_len != STRLENOF( "YYYYmmddHH:MM:SSZ" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	if ( gt.bv_val[ 10 ] != ':' || gt.bv_val[ 13 ] != ':' ) {
		return LDAP_INVALID_SYNTAX;
	}

	cnt.bv_val = ptr + 1;
	cnt.bv_len = val->bv_len - ( cnt.bv_val - val->bv_val );

	ptr = ber_bvchr( &cnt, '#' );
	if ( ptr == NULL || ptr == &val->bv_val[val->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	cnt.bv_len = ptr - cnt.bv_val;
	if ( cnt.bv_len != STRLENOF( "0x0000" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	if ( strncmp( cnt.bv_val, "0x", STRLENOF( "0x" ) ) != 0 ) {
		return LDAP_INVALID_SYNTAX;
	}

	cnt.bv_val += STRLENOF( "0x" );
	cnt.bv_len -= STRLENOF( "0x" );

	sid.bv_val = ptr + 1;
	sid.bv_len = val->bv_len - ( sid.bv_val - val->bv_val );

	ptr = ber_bvchr( &sid, '#' );
	if ( ptr == NULL || ptr == &val->bv_val[val->bv_len] ) {
		return LDAP_INVALID_SYNTAX;
	}

	sid.bv_len = ptr - sid.bv_val;
	if ( sid.bv_len != STRLENOF( "0" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	mod.bv_val = ptr + 1;
	mod.bv_len = val->bv_len - ( mod.bv_val - val->bv_val );
	if ( mod.bv_len != STRLENOF( "0000" ) ) {
		return LDAP_INVALID_SYNTAX;
	}

	bv.bv_len = STRLENOF( "YYYYmmddHHMMSS.uuuuuuZ#SSSSSS#SID#ssssss" );
	bv.bv_val = buf;

	ptr = bv.bv_val;
	ptr = lutil_strncopy( ptr, gt.bv_val, STRLENOF( "YYYYmmddHH" ) );
	ptr = lutil_strncopy( ptr, &gt.bv_val[ STRLENOF( "YYYYmmddHH:" ) ],
		STRLENOF( "MM" ) );
	ptr = lutil_strncopy( ptr, &gt.bv_val[ STRLENOF( "YYYYmmddHH:MM:" ) ],
		STRLENOF( "SS" ) );
	ptr = lutil_strcopy( ptr, ".000000Z#00" );
	ptr = lutil_strbvcopy( ptr, &cnt );
	*ptr++ = '#';
	*ptr++ = '0';
	*ptr++ = '0';
	*ptr++ = sid.bv_val[ 0 ];
	*ptr++ = '#';
	*ptr++ = '0';
	*ptr++ = '0';
	for ( i = 0; i < mod.bv_len; i++ ) {
		*ptr++ = TOLOWER( mod.bv_val[ i ] );
	}
	*ptr = '\0';

	assert( ptr == &bv.bv_val[bv.bv_len] );

	if ( csnValidate( syntax, &bv ) != LDAP_SUCCESS ) {
		return LDAP_INVALID_SYNTAX;
	}

	ber_dupbv_x( normalized, &bv, ctx );

	return LDAP_SUCCESS;
}


integerBitAndMatch(
	int *matchp,
	slap_mask_t flags,
	Syntax *syntax,
	MatchingRule *mr,
	struct berval *value,
	void *assertedValue )
{
	SLAP_LONG lValue, lAssertedValue;

	errno = 0;
	/* safe to assume integers are NUL terminated? */
	lValue = SLAP_STRTOL(value->bv_val, NULL, 10);
	if( errno == ERANGE )
	{
		return LDAP_CONSTRAINT_VIOLATION;
	}

	lAssertedValue = SLAP_STRTOL(((struct berval *)assertedValue)->bv_val,
		NULL, 10);
	if( errno == ERANGE )
	{
		return LDAP_CONSTRAINT_VIOLATION;
	}

	*matchp = ((lValue & lAssertedValue) == lAssertedValue) ? 0 : 1;
	return LDAP_SUCCESS;
}


serialNumberAndIssuerValidate(
	Syntax *syntax,
	struct berval *in )
{
	int rc;
	struct berval sn, i;

	Debug( LDAP_DEBUG_TRACE, ">>> serialNumberAndIssuerValidate: <%s>\n",
		in->bv_val, 0, 0 );

	rc = serialNumberAndIssuerCheck( in, &sn, &i, NULL );
	if ( rc ) {
		goto done;
	}

	/* validate DN -- doesn't handle double dquote */
	rc = dnValidate( NULL, &i );
	if ( rc ) {
		rc = LDAP_INVALID_SYNTAX;
	}

	if ( in->bv_val[0] == '{' && in->bv_val[in->bv_len-1] == '}' ) {
		slap_sl_free( i.bv_val, NULL );
	}

	Debug( LDAP_DEBUG_TRACE, "<<< serialNumberAndIssuerValidate: <%s> err=%d\n",
		in->bv_val, rc, 0 );

done:;
	return rc;
}


schema_destroy( void )
{
	oidm_destroy();
	oc_destroy();
	at_destroy();
	mr_destroy();
	mru_destroy();
	syn_destroy();

	if( schema_init_done ) {
		ldap_pvt_thread_mutex_destroy( &ad_index_mutex );
		ldap_pvt_thread_mutex_destroy( &ad_undef_mutex );
		ldap_pvt_thread_mutex_destroy( &oc_undef_mutex );
	}
}


static void dp8393x_update_irq(dp8393xState *s)
{
    int level = (s->regs[SONIC_IMR] & s->regs[SONIC_ISR]) ? 1 : 0;

#ifdef DEBUG_SONIC
    if (level != s->irq_level) {
        s->irq_level = level;
        if (level) {
            DPRINTF("raise irq, isr is 0x%04x\n", s->regs[SONIC_ISR]);
        } else {
            DPRINTF("lower irq\n");
        }
    }
#endif

    qemu_set_irq(s->irq, level);
}


static int dp8393x_receive_filter(dp8393xState *s, const uint8_t * buf,
                                  int size)
{
    static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
    int i;

    /* Check promiscuous mode */
    if ((s->regs[SONIC_RCR] & SONIC_RCR_PRO) && (buf[0] & 1) == 0) {
        return 0;
    }

    /* Check multicast packets */
    if ((s->regs[SONIC_RCR] & SONIC_RCR_AMC) && (buf[0] & 1) == 1) {
        return SONIC_RCR_MC;
    }

    /* Check broadcast */
    if ((s->regs[SONIC_RCR] & SONIC_RCR_BRD) && !memcmp(buf, bcast, sizeof(bcast))) {
        return SONIC_RCR_BC;
    }

    /* Check CAM */
    for (i = 0; i < 16; i++) {
        if (s->regs[SONIC_CE] & (1 << i)) {
             /* Entry enabled */
             if (!memcmp(buf, s->cam[i], sizeof(s->cam[i]))) {
                 return 0;
             }
        }
    }

    return -1;
}


static void dp8393x_do_receiver_disable(dp8393xState *s)
{
    s->regs[SONIC_CR] &= ~SONIC_CR_RXEN;
}


static void dp8393x_update_wt_regs(dp8393xState *s)
{
    int64_t elapsed;
    uint32_t val;

    if (s->regs[SONIC_CR] & SONIC_CR_STP) {
        timer_del(s->watchdog);
        return;
    }

    elapsed = s->wt_last_update - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
    val = dp8393x_wt(s);
    val -= elapsed / 5000000;
    s->regs[SONIC_WT1] = (val >> 16) & 0xffff;
    s->regs[SONIC_WT0] = (val >> 0)  & 0xffff;
    dp8393x_set_next_tick(s);

}


void HtmlPage::addChar(GfxState *state, double x, double y,
		       double dx, double dy,
			double ox, double oy, Unicode *u, int uLen) {
  double x1, y1, w1, h1, dx2, dy2;
  int n, i;
  state->transform(x, y, &x1, &y1);
  n = curStr->len;

  // check that new character is in the same direction as current string
  // and is not too far away from it before adding
  //if ((UnicodeMap::getDirection(u[0]) != curStr->dir) ||
  // XXX
  if (debug) {
    const double *text_mat = state->getTextMat();
    // rotation is (cos q, sin q, -sin q, cos q, 0, 0)
    // sin q is zero iff there is no rotation, or 180 deg. rotation;
    // for 180 rotation, cos q will be negative
    if (text_mat[0] < 0 || !is_within(text_mat[1], .1, 0)) {
      std::cerr << DEBUG << "rotation matrix for \"" << print_uni_str(u, uLen) << '"' << std::endl;
      std::cerr << "text " << print_matrix(state->getTextMat());
    }
  }
  if (n > 0 && // don't start a new string, unless there is already a string
      // TODO: the following line assumes that text is flowing left to
      // right, which will not necessarily be the case, e.g. if rotated;
      // It assesses whether or not two characters are close enough to
      // be part of the same string
      fabs(x1 - curStr->xRight[n-1]) > wordBreakThreshold * (curStr->yMax - curStr->yMin) &&
      // rotation is (cos q, sin q, -sin q, cos q, 0, 0)
      // sin q is zero iff there is no rotation, or 180 deg. rotation;
      // for 180 rotation, cos q will be negative
      !rot_matrices_equal(curStr->getFont().getRotMat(), state->getTextMat()))
  {
    endString();
    beginString(state, nullptr);
  }
  state->textTransformDelta(state->getCharSpace() * state->getHorizScaling(),
			    0, &dx2, &dy2);
  dx -= dx2;
  dy -= dy2;
  state->transformDelta(dx, dy, &w1, &h1);
  if (uLen != 0) {
    w1 /= uLen;
    h1 /= uLen;
  }
  for (i = 0; i < uLen; ++i) {
    curStr->addChar(state, x1 + i*w1, y1 + i*h1, w1, h1, u[i]);
  }
}


static GooString* Dirname(GooString* str){

  char *p=str->c_str();
  int len=str->getLength();
  for (int i=len-1;i>=0;i--)
    if (*(p+i)==SLASH)
      return new GooString(p,i+1);
  return new GooString();
}


intrusive_ptr<Expression> ExpressionNary::optimize() {
    uint32_t constOperandCount = 0;

    for (auto& operand : vpOperand) {
        operand = operand->optimize();
        if (dynamic_cast<ExpressionConstant*>(operand.get())) {
            ++constOperandCount;
        }
    }
    // If all the operands are constant expressions, collapse the expression into one constant
    // expression.
    if (constOperandCount == vpOperand.size()) {
        return intrusive_ptr<Expression>(
            ExpressionConstant::create(getExpressionContext(), evaluate(Document())));
    }

    // If the expression is associative, we can collapse all the consecutive constant operands into
    // one by applying the expression to those consecutive constant operands.
    // If the expression is also commutative we can reorganize all the operands so that all of the
    // constant ones are together (arbitrarily at the back) and we can collapse all of them into
    // one.
    if (isAssociative()) {
        ExpressionVector constExpressions;
        ExpressionVector optimizedOperands;
        for (size_t i = 0; i < vpOperand.size();) {
            intrusive_ptr<Expression> operand = vpOperand[i];
            // If the operand is a constant one, add it to the current list of consecutive constant
            // operands.
            if (dynamic_cast<ExpressionConstant*>(operand.get())) {
                constExpressions.push_back(operand);
                ++i;
                continue;
            }

            // If the operand is exactly the same type as the one we are currently optimizing and
            // is also associative, replace the expression for the operands it has.
            // E.g: sum(a, b, sum(c, d), e) => sum(a, b, c, d, e)
            ExpressionNary* nary = dynamic_cast<ExpressionNary*>(operand.get());
            if (nary && str::equals(nary->getOpName(), getOpName()) && nary->isAssociative()) {
                invariant(!nary->vpOperand.empty());
                vpOperand[i] = std::move(nary->vpOperand[0]);
                vpOperand.insert(
                    vpOperand.begin() + i + 1, nary->vpOperand.begin() + 1, nary->vpOperand.end());
                continue;
            }

            // If the operand is not a constant nor a same-type expression and the expression is
            // not commutative, evaluate an expression of the same type as the one we are
            // optimizing on the list of consecutive constant operands and use the resulting value
            // as a constant expression operand.
            // If the list of consecutive constant operands has less than 2 operands just place
            // back the operands.
            if (!isCommutative()) {
                if (constExpressions.size() > 1) {
                    ExpressionVector vpOperandSave = std::move(vpOperand);
                    vpOperand = std::move(constExpressions);
                    optimizedOperands.emplace_back(
                        ExpressionConstant::create(getExpressionContext(), evaluate(Document())));
                    vpOperand = std::move(vpOperandSave);
                } else {
                    optimizedOperands.insert(
                        optimizedOperands.end(), constExpressions.begin(), constExpressions.end());
                }
                constExpressions.clear();
            }
            optimizedOperands.push_back(operand);
            ++i;
        }

        if (constExpressions.size() > 1) {
            vpOperand = std::move(constExpressions);
            optimizedOperands.emplace_back(
                ExpressionConstant::create(getExpressionContext(), evaluate(Document())));
        } else {
            optimizedOperands.insert(
                optimizedOperands.end(), constExpressions.begin(), constExpressions.end());
        }

        vpOperand = std::move(optimizedOperands);
    }
    return this;
}


    void run() {
        intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest());
        intrusive_ptr<Expression> expression = ExpressionConstant::create(expCtx, Value(5));
        DepsTracker dependencies;
        expression->addDependencies(&dependencies);
        ASSERT_EQUALS(0U, dependencies.fields.size());
        ASSERT_EQUALS(false, dependencies.needWholeDocument);
        ASSERT_EQUALS(false, dependencies.getNeedTextScore());
    }


TEST(BuiltinRemoveVariableTest, LiteralEscapesRemoveVar) {
    assertExpectedResults(
        "$literal", {{{Value("$$REMOVE"_sd)}, Value(std::vector<Value>{Value("$$REMOVE"_sd)})}});
}


TEST(ExpressionTypeTest, WithOIDValue) {
    assertExpectedResults("$type", {{{Value(OID())}, Value("objectId"_sd)}});
}


Value setIsSubsetHelper(const vector<Value>& lhs, const ValueSet& rhs) {
    // do not shortcircuit when lhs.size() > rhs.size()
    // because lhs can have redundant entries
    for (vector<Value>::const_iterator it = lhs.begin(); it != lhs.end(); ++it) {
        if (!rhs.count(*it)) {
            return Value(false);
        }
    }
    return Value(true);
}


Document fromBson(BSONObj obj) {
    return Document(obj);
}


    void run() {
        intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest());
        const Document spec = getSpec();
        const Value args = spec["input"];
        if (!spec["expected"].missing()) {
            FieldIterator fields(spec["expected"].getDocument());
            while (fields.more()) {
                const Document::FieldPair field(fields.next());
                const Value expected = field.second;
                const BSONObj obj = BSON(field.first << args);
                VariablesParseState vps = expCtx->variablesParseState;
                const intrusive_ptr<Expression> expr =
                    Expression::parseExpression(expCtx, obj, vps);
                const Value result = expr->evaluate(Document());
                if (ValueComparator().evaluate(result != expected)) {
                    string errMsg = str::stream()
                        << "for expression " << field.first.toString() << " with argument "
                        << args.toString() << " full tree: " << expr->serialize(false).toString()
                        << " expected: " << expected.toString()
                        << " but got: " << result.toString();
                    FAIL(errMsg);
                }
                // TODO test optimize here
            }
        }
        if (!spec["error"].missing()) {
            const vector<Value>& asserters = spec["error"].getArray();
            size_t n = asserters.size();
            for (size_t i = 0; i < n; i++) {
                const BSONObj obj = BSON(asserters[i].getString() << args);
                VariablesParseState vps = expCtx->variablesParseState;
                ASSERT_THROWS(
                    {
                        // NOTE: parse and evaluatation failures are treated the
                        // same
                        const intrusive_ptr<Expression> expr =
                            Expression::parseExpression(expCtx, obj, vps);
                        expr->evaluate(Document());
                    },
                    AssertionException);
            }
        }
    }


TEST(ExpressionTypeTest, WithSymbolValue) {
    assertExpectedResults("$type", {{{Value(BSONSymbol("a"))}, Value("symbol"_sd)}});
}


Value ExpressionFieldPath::evaluate(const Document& root) const {
    auto& vars = getExpressionContext()->variables;
    if (_fieldPath.getPathLength() == 1)  // get the whole variable
        return vars.getValue(_variable, root);

    if (_variable == Variables::kRootId) {
        // ROOT is always a document so use optimized code path
        return evaluatePath(1, root);
    }

    Value var = vars.getValue(_variable, root);
    switch (var.getType()) {
        case Object:
            return evaluatePath(1, var.getDocument());
        case Array:
            return evaluatePathArray(1, var);
        default:
            return Value();
    }
}


TEST(ExpressionArrayToObjectTest, ListFormWithDuplicates) {
    assertExpectedResults("$arrayToObject",
                          {{{Value(BSON_ARRAY(BSON_ARRAY("key1" << 2) << BSON_ARRAY("key1" << 3)))},
                            {Value(BSON("key1" << 3))}}});
}


void ExpressionDateFromString::_doAddDependencies(DepsTracker* deps) const {
    _dateString->addDependencies(deps);
    if (_timeZone) {
        _timeZone->addDependencies(deps);
    }
}