Link to PDF file (or attach file here):
[pages4.pdf](https://github.com/mozilla/pdf.js/files/1128300/pages4.pdf)

Configuration:
- Web browser and its version: Firefox 54
- Operating system and its version: Win 10
- PDF.js version: 1.8.522
- Is an extension: no

What is the expected behavior? (add screenshot)
![chrome1-1 1](https://user-images.githubusercontent.com/146436/27918380-2f74512a-6245-11e7-8525-d0038fa6c2e8.jpg)
![chrome1-2 1](https://user-images.githubusercontent.com/146436/27918425-54f8e47e-6245-11e7-9bff-127f79ad0511.jpg)


What went wrong? (add screenshot)
![screenshot from 2017-07-06 12-10-30](https://user-images.githubusercontent.com/146436/27918442-5d57624e-6245-11e7-9436-5ba31cc8bd1a.png)
![screenshot from 2017-07-06 12-10-44](https://user-images.githubusercontent.com/146436/27918454-63b486da-6245-11e7-9eb5-c03aa5c40943.png)



const length = maskData.length;
  const scale = 1 / 255;
  for (let i = 3; i < length; i += 4) {
    const alpha = transferMap ? transferMap[maskData[i]] : maskData[i];
    layerData[i] = (layerData[i] * alpha * scale) | 0;
  }
}
function composeSMaskLuminosity(maskData, layerData, transferMap) {
  const length = maskData.length;
  for (let i = 3; i < length; i += 4) {
    const y =
      maskData[i - 3] * 77 + 
      maskData[i - 2] * 152 + 
      maskData[i - 1] * 28; 
    layerData[i] = transferMap
      ? (layerData[i] * transferMap[y >> 8]) >> 8
      : (layerData[i] * y) >> 16;
  }
}
function genericComposeSMask(