Attach (recommended) or Link to PDF file here:
[PATTYP2.pdf](https://github.com/mozilla/pdf.js/files/6471935/PATTYP2.pdf)

Configuration:
- Web browser and its version: FF 88.0.1
- Operating system and its version: W10.0.19042.985
- PDF.js version: https://mozilla.github.io/pdf.js/web/viewer.html

Steps to reproduce the problem:
1. open file to display

What is the expected behavior? (add screenshot)
![grafik](https://user-images.githubusercontent.com/6665575/118114800-5e407800-b3e8-11eb-860f-019f48a8eabf.png)

What went wrong? (add screenshot)
![grafik](https://user-images.githubusercontent.com/6665575/118114841-6c8e9400-b3e8-11eb-8252-1517e5d76475.png)

(memo for me: when fixed, check the files in https://issues.apache.org/jira/browse/PDFBOX-1958 )

const length = maskData.length;
    const scale = 1 / 255;
    for (let i = 3; i < length; i += 4) {
      const alpha = transferMap ? transferMap[maskData[i]] : maskData[i];
      layerData[i] = (layerData[i] * alpha * scale) | 0;
    }
  }
  function composeSMaskLuminosity(maskData, layerData, transferMap) {
    const length = maskData.length;
    for (let i = 3; i < length; i += 4) {
      const y =
        maskData[i - 3] * 77 + 
        maskData[i - 2] * 152 + 
        maskData[i - 1] * 28; 
      layerData[i] = transferMap
        ? (layerData[i] * transferMap[y >> 8]) >> 8
        : (layerData[i] * y) >> 16;
    }
  }
  function genericComposeSMask(
    maskCtx,
    layerCtx,
    width,