content: /*
   * Copyright (C) 2011 The Android Open Source Project
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  
  package android.filterpacks.videosrc;
  
  import android.content.Context;
  import android.content.res.AssetFileDescriptor;
  import android.filterfw.core.Filter;
  import android.filterfw.core.FilterContext;
  import android.filterfw.core.Frame;
  import android.filterfw.core.FrameFormat;
  import android.filterfw.core.GenerateFieldPort;
  import android.filterfw.core.GenerateFinalPort;
  import android.filterfw.core.GLFrame;
  import android.filterfw.core.MutableFrameFormat;
  import android.filterfw.core.ShaderProgram;
  import android.filterfw.format.ImageFormat;
  import android.graphics.SurfaceTexture;
  import android.media.MediaPlayer;
  import android.net.Uri;
  import android.opengl.Matrix;
  import android.view.Surface;
  
  import java.io.IOException;
  import java.lang.IllegalArgumentException;
  
  import android.util.Log;
  
  /**
   * @hide
   */
  public class MediaSource extends Filter {
  
      /** User-visible parameters */
  
      /** The source URL for the media source. Can be an http: link to a remote
       * resource, or a file: link to a local media file
       */
      @GenerateFieldPort(name = "sourceUrl", hasDefault = true)
      private String mSourceUrl = "";
  
      /** An open asset file descriptor to a local media source. Default is null */
      @GenerateFieldPort(name = "sourceAsset", hasDefault = true)
      private AssetFileDescriptor mSourceAsset = null;
  
      /** The context for the MediaPlayer to resolve the sourceUrl.
       * Make sure this is set before the sourceUrl to avoid unexpected result.
       * If the sourceUrl is not a content URI, it is OK to keep this as null. */
      @GenerateFieldPort(name = "context", hasDefault = true)
      private Context mContext = null;
  
      /** Whether the media source is a URL or an asset file descriptor. Defaults
       * to false.
       */
      @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true)
      private boolean mSelectedIsUrl = false;
  
      /** Whether the filter will always wait for a new video frame, or whether it
       * will output an old frame again if a new frame isn't available. Defaults
       * to true.
       */
      @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
      private boolean mWaitForNewFrame = true;
  
      /** Whether the media source should loop automatically or not. Defaults to
       * true.
       */
      @GenerateFieldPort(name = "loop", hasDefault = true)
      private boolean mLooping = true;
  
      /** Volume control. Currently sound is piped directly to the speakers, so
       * this defaults to mute.
       */
      @GenerateFieldPort(name = "volume", hasDefault = true)
      private float mVolume = 0.f;
  
      /** Orientation. This controls the output orientation of the video. Valid
       * values are 0, 90, 180, 270
       */
      @GenerateFieldPort(name = "orientation", hasDefault = true)
      private int mOrientation = 0;
  
      private MediaPlayer mMediaPlayer;
      private GLFrame mMediaFrame;
      private SurfaceTexture mSurfaceTexture;
      private ShaderProgram mFrameExtractor;
      private MutableFrameFormat mOutputFormat;
      private int mWidth, mHeight;
  
      // Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT
      private static final int PREP_TIMEOUT = 100; // ms
      private static final int PREP_TIMEOUT_REPEAT = 100;
      private static final int NEWFRAME_TIMEOUT = 100; //ms
      private static final int NEWFRAME_TIMEOUT_REPEAT = 10;
  
      // This is an identity shader; not using the default identity
      // shader because reading from a SurfaceTexture requires the
      // GL_OES_EGL_image_external extension.
      private final String mFrameShader =
              "#extension GL_OES_EGL_image_external : require\n" +
              "precision mediump float;\n" +
              "uniform samplerExternalOES tex_sampler_0;\n" +
              "varying vec2 v_texcoord;\n" +
              "void main() {\n" +
              "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
              "}\n";
  
      // The following transforms enable rotation of the decoded source.
      // These are multiplied with the transform obtained from the
      // SurfaceTexture to get the final transform to be set on the media source.
      // Currently, given a device orientation, the MediaSource rotates in such a way
      // that the source is displayed upright. A particular use case
      // is "Background Replacement" feature in the Camera app
      // where the MediaSource rotates the source to align with the camera feed and pass it
      // on to the backdropper filter. The backdropper only does the blending
      // and does not have to do any rotation
      // (except for mirroring in case of front camera).
      // TODO: Currently the rotations are spread over a bunch of stages in the
      // pipeline. A cleaner design
      // could be to cast away all the rotation in a separate filter or attach a transform
      // to the frame so that MediaSource itself need not know about any rotation.
      private static final float[] mSourceCoords_0 = { 1, 1, 0, 1,
                                                       0, 1, 0, 1,
                                                       1, 0, 0, 1,
                                                       0, 0, 0, 1 };
      private static final float[] mSourceCoords_270 = { 0, 1, 0, 1,
                                                        0, 0, 0, 1,
                                                        1, 1, 0, 1,
                                                        1, 0, 0, 1 };
      private static final float[] mSourceCoords_180 = { 0, 0, 0, 1,
                                                         1, 0, 0, 1,
                                                         0, 1, 0, 1,
                                                         1, 1, 0, 1 };
      private static final float[] mSourceCoords_90 = { 1, 0, 0, 1,
                                                         1, 1, 0, 1,
                                                         0, 0, 0, 1,
                                                         0, 1, 0, 1 };
  
      private boolean mGotSize;
      private boolean mPrepared;
      private boolean mPlaying;
      private boolean mNewFrameAvailable;
      private boolean mOrientationUpdated;
      private boolean mPaused;
      private boolean mCompleted;
  
      private final boolean mLogVerbose;
      private static final String TAG = "MediaSource";
  
      public MediaSource(String name) {
          super(name);
          mNewFrameAvailable = false;
  
          mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
      }
  
      @Override
      public void setupPorts() {
          // Add input port
          addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                    FrameFormat.TARGET_GPU));
      }
  
      private void createFormats() {
          mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                             FrameFormat.TARGET_GPU);
      }
  
      @Override
      protected void prepare(FilterContext context) {
          if (mLogVerbose) Log.v(TAG, "Preparing MediaSource");
  
          mFrameExtractor = new ShaderProgram(context, mFrameShader);
          // SurfaceTexture defines (0,0) to be bottom-left. The filter framework
          // defines (0,0) as top-left, so do the flip here.
          mFrameExtractor.setSourceRect(0, 1, 1, -1);
  
          createFormats();
      }
  
      @Override
      public void open(FilterContext context) {
          if (mLogVerbose) {
              Log.v(TAG, "Opening MediaSource");
              if (mSelectedIsUrl) {
                  Log.v(TAG, "Current URL is " + mSourceUrl);
              } else {
                  Log.v(TAG, "Current source is Asset!");
              }
          }
  
          mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(
                  mOutputFormat,
                  GLFrame.EXTERNAL_TEXTURE,
                  0);
  
          mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
  
          if (!setupMediaPlayer(mSelectedIsUrl)) {
            throw new RuntimeException("Error setting up MediaPlayer!");
          }
      }
  
      @Override
      public void process(FilterContext context) {
          // Note: process is synchronized by its caller in the Filter base class
          if (mLogVerbose) Log.v(TAG, "Processing new frame");
  
          if (mMediaPlayer == null) {
              // Something went wrong in initialization or parameter updates
              throw new NullPointerException("Unexpected null media player!");
          }
  
          if (mCompleted) {
              // Video playback is done, so close us down
              closeOutputPort("video");
              return;
          }
  
          if (!mPlaying) {
              int waitCount = 0;
              if (mLogVerbose) Log.v(TAG, "Waiting for preparation to complete");
              while (!mGotSize || !mPrepared) {
                  try {
                      this.wait(PREP_TIMEOUT);
                  } catch (InterruptedException e) {
                      // ignoring
                  }
                  if (mCompleted) {
                      // Video playback is done, so close us down
                      closeOutputPort("video");
                      return;
                  }
                  waitCount++;
                  if (waitCount == PREP_TIMEOUT_REPEAT) {
                      mMediaPlayer.release();
                      throw new RuntimeException("MediaPlayer timed out while preparing!");
                  }
              }
              if (mLogVerbose) Log.v(TAG, "Starting playback");
              mMediaPlayer.start();
          }
  
          // Use last frame if paused, unless just starting playback, in which case
          // we want at least one valid frame before pausing
          if (!mPaused || !mPlaying) {
              if (mWaitForNewFrame) {
                  if (mLogVerbose) Log.v(TAG, "Waiting for new frame");
  
                  int waitCount = 0;
                  while (!mNewFrameAvailable) {
                      if (waitCount == NEWFRAME_TIMEOUT_REPEAT) {
                          if (mCompleted) {
                              // Video playback is done, so close us down
                              closeOutputPort("video");
                              return;
                          } else {
                              throw new RuntimeException("Timeout waiting for new frame!");
                          }
                      }
                      try {
                          this.wait(NEWFRAME_TIMEOUT);
                      } catch (InterruptedException e) {
                          if (mLogVerbose) Log.v(TAG, "interrupted");
                          // ignoring
                      }
                      waitCount++;
                  }
                  mNewFrameAvailable = false;
                  if (mLogVerbose) Log.v(TAG, "Got new frame");
              }
  
              mSurfaceTexture.updateTexImage();
              mOrientationUpdated = true;
          }
          if (mOrientationUpdated) {
              float[] surfaceTransform = new float[16];
              mSurfaceTexture.getTransformMatrix(surfaceTransform);
  
              float[] sourceCoords = new float[16];
              switch (mOrientation) {
                  default:
                  case 0:
                      Matrix.multiplyMM(sourceCoords, 0,
                                        surfaceTransform, 0,
                                        mSourceCoords_0, 0);
                      break;
                  case 90:
                      Matrix.multiplyMM(sourceCoords, 0,
                                        surfaceTransform, 0,
                                        mSourceCoords_90, 0);
                      break;
                  case 180:
                      Matrix.multiplyMM(sourceCoords, 0,
                                        surfaceTransform, 0,
                                        mSourceCoords_180, 0);
                      break;
                  case 270:
                      Matrix.multiplyMM(sourceCoords, 0,
                                        surfaceTransform, 0,
                                        mSourceCoords_270, 0);
                      break;
              }
              if (mLogVerbose) {
                  Log.v(TAG, "OrientationHint = " + mOrientation);
                  String temp = String.format("SetSourceRegion: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f",
                          sourceCoords[4], sourceCoords[5],sourceCoords[0], sourceCoords[1],
                          sourceCoords[12], sourceCoords[13],sourceCoords[8], sourceCoords[9]);
                  Log.v(TAG, temp);
              }
              mFrameExtractor.setSourceRegion(sourceCoords[4], sourceCoords[5],
                      sourceCoords[0], sourceCoords[1],
                      sourceCoords[12], sourceCoords[13],
                      sourceCoords[8], sourceCoords[9]);
              mOrientationUpdated = false;
          }
  
          Frame output = context.getFrameManager().newFrame(mOutputFormat);
          mFrameExtractor.process(mMediaFrame, output);
  
          long timestamp = mSurfaceTexture.getTimestamp();
          if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
          output.setTimestamp(timestamp);
  
          pushOutput("video", output);
          output.release();
  
          mPlaying = true;
      }
  
      @Override
      public void close(FilterContext context) {
          if (mMediaPlayer.isPlaying()) {
              mMediaPlayer.stop();
          }
          mPrepared = false;
          mGotSize = false;
          mPlaying = false;
          mPaused = false;
          mCompleted = false;
          mNewFrameAvailable = false;
  
          mMediaPlayer.release();
          mMediaPlayer = null;
          mSurfaceTexture.release();
          mSurfaceTexture = null;
          if (mLogVerbose) Log.v(TAG, "MediaSource closed");
      }
  
      @Override
      public void tearDown(FilterContext context) {
          if (mMediaFrame != null) {
              mMediaFrame.release();
          }
      }
  
      // When updating the port values of the filter, users can update sourceIsUrl to switch
      //   between using URL objects or Assets.
      // If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player
      //   uses Url objects/Asset.
      // Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch
      //   sourceIsUrl next time.
      @Override
      public void fieldPortValueUpdated(String name, FilterContext context) {
          if (mLogVerbose) Log.v(TAG, "Parameter update");
          if (name.equals("sourceUrl")) {
             if (isOpen()) {
                  if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                  if (mSelectedIsUrl) {
                      setupMediaPlayer(mSelectedIsUrl);
                  }
              }
          } else if (name.equals("sourceAsset") ) {
              if (isOpen()) {
                  if (mLogVerbose) Log.v(TAG, "Opening new source FD");
                  if (!mSelectedIsUrl) {
                      setupMediaPlayer(mSelectedIsUrl);
                  }
              }
          } else if (name.equals("loop")) {
              if (isOpen()) {
                  mMediaPlayer.setLooping(mLooping);
              }
          } else if (name.equals("sourceIsUrl")) {
              if (isOpen()){
                  if (mSelectedIsUrl){
                      if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                  } else {
                      if (mLogVerbose) Log.v(TAG, "Opening new source Asset");
                  }
                  setupMediaPlayer(mSelectedIsUrl);
              }
          } else if (name.equals("volume")) {
              if (isOpen()) {
                  mMediaPlayer.setVolume(mVolume, mVolume);
              }
          } else if (name.equals("orientation") && mGotSize) {
              if (mOrientation == 0 || mOrientation == 180) {
                  mOutputFormat.setDimensions(mWidth, mHeight);
              } else {
                  mOutputFormat.setDimensions(mHeight, mWidth);
              }
              mOrientationUpdated = true;
          }
      }
  
      synchronized public void pauseVideo(boolean pauseState) {
          if (isOpen()) {
              if (pauseState && !mPaused) {
                  mMediaPlayer.pause();
              } else if (!pauseState && mPaused) {
                  mMediaPlayer.start();
              }
          }
          mPaused = pauseState;
      }
  
      /** Creates a media player, sets it up, and calls prepare */
      synchronized private boolean setupMediaPlayer(boolean useUrl) {
          mPrepared = false;
          mGotSize = false;
          mPlaying = false;
          mPaused = false;
          mCompleted = false;
          mNewFrameAvailable = false;
  
          if (mLogVerbose) Log.v(TAG, "Setting up playback.");
  
          if (mMediaPlayer != null) {
              // Clean up existing media players
              if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer.");
              mMediaPlayer.reset();
          } else {
              // Create new media player
              if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer.");
              mMediaPlayer = new MediaPlayer();
          }
  
          if (mMediaPlayer == null) {
              throw new RuntimeException("Unable to create a MediaPlayer!");
          }
  
          // Set up data sources, etc
          try {
              if (useUrl) {
                  if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl);
                  if (mContext == null) {
                      mMediaPlayer.setDataSource(mSourceUrl);
                  } else {
                      mMediaPlayer.setDataSource(mContext, Uri.parse(mSourceUrl.toString()));
                  }
              } else {
                  if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset);
                  mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength());
              }
          } catch(IOException e) {
              mMediaPlayer.release();
              mMediaPlayer = null;
              if (useUrl) {
                  throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
              } else {
                  throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
              }
          } catch(IllegalArgumentException e) {
              mMediaPlayer.release();
              mMediaPlayer = null;
              if (useUrl) {
                  throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
              } else {
                  throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
              }
          }
  
          mMediaPlayer.setLooping(mLooping);
          mMediaPlayer.setVolume(mVolume, mVolume);
  
          // Bind it to our media frame
          Surface surface = new Surface(mSurfaceTexture);
          mMediaPlayer.setSurface(surface);
          surface.release();
  
          // Connect Media Player to callbacks
  
          mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener);
          mMediaPlayer.setOnPreparedListener(onPreparedListener);
          mMediaPlayer.setOnCompletionListener(onCompletionListener);
  
          // Connect SurfaceTexture to callback
          mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener);
  
          if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer.");
          mMediaPlayer.prepareAsync();
  
          return true;
      }
  
      private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener =
              new MediaPlayer.OnVideoSizeChangedListener() {
          public void onVideoSizeChanged(MediaPlayer mp, int width, int height) {
              if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height);
              if (!mGotSize) {
                  if (mOrientation == 0 || mOrientation == 180) {
                      mOutputFormat.setDimensions(width, height);
                  } else {
                      mOutputFormat.setDimensions(height, width);
                  }
                  mWidth = width;
                  mHeight = height;
              } else {
                  if (mOutputFormat.getWidth() != width ||
                      mOutputFormat.getHeight() != height) {
                      Log.e(TAG, "Multiple video size change events received!");
                  }
              }
              synchronized(MediaSource.this) {
                  mGotSize = true;
                  MediaSource.this.notify();
              }
          }
      };
  
      private MediaPlayer.OnPreparedListener onPreparedListener =
              new MediaPlayer.OnPreparedListener() {
          public void onPrepared(MediaPlayer mp) {
              if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared");
              synchronized(MediaSource.this) {
                  mPrepared = true;
                  MediaSource.this.notify();
              }
          }
      };
  
      private MediaPlayer.OnCompletionListener onCompletionListener =
              new MediaPlayer.OnCompletionListener() {
          public void onCompletion(MediaPlayer mp) {
              if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback");
              synchronized(MediaSource.this) {
                  mCompleted = true;
              }
          }
      };
  
      private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener =
              new SurfaceTexture.OnFrameAvailableListener() {
          public void onFrameAvailable(SurfaceTexture surfaceTexture) {
              if (mLogVerbose) Log.v(TAG, "New frame from media player");
              synchronized(MediaSource.this) {
                  if (mLogVerbose) Log.v(TAG, "New frame: notify");
                  mNewFrameAvailable = true;
                  MediaSource.this.notify();
                  if (mLogVerbose) Log.v(TAG, "New frame: notify done");
              }
          }
      };
  
  }
  