Attach (recommended) or Link to PDF file here:
[Test file.pdf](https://github.com/mozilla/pdf.js/files/4656226/Test.file.pdf)

Configuration:
- Web browser and its version: Chrome Version 81.0.4044.138 (Official Build) (64-bit)
- Operating system and its version: Windows 10
- PDF.js version: latest
- Is a browser extension: no

Steps to reproduce the problem:
1. Open [https://mozilla.github.io/pdf.js/web/viewer.html](https://mozilla.github.io/pdf.js/web/viewer.html)
2. Load the provided .pdf file

What is the expected behavior? (add screenshot)
![image](https://user-images.githubusercontent.com/5569382/82438616-02c3fe80-9aa2-11ea-9003-b75ea1e7aac6.png)

What went wrong? (add screenshot)
![image](https://user-images.githubusercontent.com/5569382/82438640-09527600-9aa2-11ea-810b-071da7e37896.png)


const length = maskData.length;
    for (let i = 3; i < length; i += 4) {
      const y =
        maskData[i - 3] * 77 + 
        maskData[i - 2] * 152 + 
        maskData[i - 1] * 28; 
      layerData[i] = transferMap
        ? (layerData[i] * transferMap[y >> 8]) >> 8
        : (layerData[i] * y) >> 16;
    }
  }
  function genericComposeSMask(
    maskCtx,
    layerCtx,
    width,
    height,
    subtype,
    backdrop,
    transferMap
  ) {
    const hasBackdrop = !!backdrop;
    const r0 = hasBackdrop ? backdrop[0] : 0;
    const g0 = hasBackdrop ? backdrop[1] : 0;
    const b0 = hasBackdrop ? backdrop[2] : 0;
    let composeFn;
    if (subtype === "Luminosity") {