content: /*
   * Copyright (C) 2018 The Android Open Source Project
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *      http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package com.android.server;
  
  import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
  
  import static java.lang.annotation.RetentionPolicy.SOURCE;
  
  import android.annotation.IntDef;
  import android.annotation.Nullable;
  import android.content.Context;
  import android.content.pm.PackageManager;
  import android.content.pm.VersionedPackage;
  import android.net.NetworkStackClient;
  import android.os.Environment;
  import android.os.Handler;
  import android.os.Looper;
  import android.os.SystemClock;
  import android.provider.DeviceConfig;
  import android.text.TextUtils;
  import android.util.ArrayMap;
  import android.util.ArraySet;
  import android.util.AtomicFile;
  import android.util.Slog;
  import android.util.Xml;
  
  import com.android.internal.annotations.GuardedBy;
  import com.android.internal.annotations.VisibleForTesting;
  import com.android.internal.os.BackgroundThread;
  import com.android.internal.util.FastXmlSerializer;
  import com.android.internal.util.XmlUtils;
  
  import libcore.io.IoUtils;
  
  import org.xmlpull.v1.XmlPullParser;
  import org.xmlpull.v1.XmlPullParserException;
  import org.xmlpull.v1.XmlSerializer;
  
  import java.io.File;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.lang.annotation.Retention;
  import java.lang.annotation.RetentionPolicy;
  import java.nio.charset.StandardCharsets;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.concurrent.TimeUnit;
  
  /**
   * Monitors the health of packages on the system and notifies interested observers when packages
   * fail. On failure, the registered observer with the least user impacting mitigation will
   * be notified.
   */
  public class PackageWatchdog {
      private static final String TAG = "PackageWatchdog";
  
      static final String PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS =
              "watchdog_trigger_failure_duration_millis";
      static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =
              "watchdog_trigger_failure_count";
      static final String PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED =
              "watchdog_explicit_health_check_enabled";
  
      public static final int FAILURE_REASON_UNKNOWN = 0;
      public static final int FAILURE_REASON_NATIVE_CRASH = 1;
      public static final int FAILURE_REASON_EXPLICIT_HEALTH_CHECK = 2;
      public static final int FAILURE_REASON_APP_CRASH = 3;
      public static final int FAILURE_REASON_APP_NOT_RESPONDING = 4;
  
      @IntDef(prefix = { "FAILURE_REASON_" }, value = {
              FAILURE_REASON_UNKNOWN,
              FAILURE_REASON_NATIVE_CRASH,
              FAILURE_REASON_EXPLICIT_HEALTH_CHECK,
              FAILURE_REASON_APP_CRASH,
              FAILURE_REASON_APP_NOT_RESPONDING
      })
      @Retention(RetentionPolicy.SOURCE)
      public @interface FailureReasons {}
  
      // Duration to count package failures before it resets to 0
      private static final int DEFAULT_TRIGGER_FAILURE_DURATION_MS =
              (int) TimeUnit.MINUTES.toMillis(1);
      // Number of package failures within the duration above before we notify observers
      private static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5;
      // Whether explicit health checks are enabled or not
      private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true;
  
      private static final int DB_VERSION = 1;
      private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog";
      private static final String TAG_PACKAGE = "package";
      private static final String TAG_OBSERVER = "observer";
      private static final String ATTR_VERSION = "version";
      private static final String ATTR_NAME = "name";
      private static final String ATTR_DURATION = "duration";
      private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
      private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
  
      @GuardedBy("PackageWatchdog.class")
      private static PackageWatchdog sPackageWatchdog;
  
      private final Object mLock = new Object();
      // System server context
      private final Context mContext;
      // Handler to run short running tasks
      private final Handler mShortTaskHandler;
      // Handler for processing IO and long running tasks
      private final Handler mLongTaskHandler;
      // Contains (observer-name -> observer-handle) that have ever been registered from
      // previous boots. Observers with all packages expired are periodically pruned.
      // It is saved to disk on system shutdown and repouplated on startup so it survives reboots.
      @GuardedBy("mLock")
      private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>();
      // File containing the XML data of monitored packages /data/system/package-watchdog.xml
      private final AtomicFile mPolicyFile;
      private final ExplicitHealthCheckController mHealthCheckController;
      private final NetworkStackClient mNetworkStackClient;
      @GuardedBy("mLock")
      private boolean mIsPackagesReady;
      // Flag to control whether explicit health checks are supported or not
      @GuardedBy("mLock")
      private boolean mIsHealthCheckEnabled = DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED;
      @GuardedBy("mLock")
      private int mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS;
      @GuardedBy("mLock")
      private int mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
      // SystemClock#uptimeMillis when we last executed #syncState
      // 0 if no prune is scheduled.
      @GuardedBy("mLock")
      private long mUptimeAtLastStateSync;
  
      private PackageWatchdog(Context context) {
          // Needs to be constructed inline
          this(context, new AtomicFile(
                          new File(new File(Environment.getDataDirectory(), "system"),
                                  "package-watchdog.xml")),
                  new Handler(Looper.myLooper()), BackgroundThread.getHandler(),
                  new ExplicitHealthCheckController(context),
                  NetworkStackClient.getInstance());
      }
  
      /**
       * Creates a PackageWatchdog that allows injecting dependencies.
       */
      @VisibleForTesting
      PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler,
              Handler longTaskHandler, ExplicitHealthCheckController controller,
              NetworkStackClient networkStackClient) {
          mContext = context;
          mPolicyFile = policyFile;
          mShortTaskHandler = shortTaskHandler;
          mLongTaskHandler = longTaskHandler;
          mHealthCheckController = controller;
          mNetworkStackClient = networkStackClient;
          loadFromFile();
      }
  
      /** Creates or gets singleton instance of PackageWatchdog. */
      public static PackageWatchdog getInstance(Context context) {
          synchronized (PackageWatchdog.class) {
              if (sPackageWatchdog == null) {
                  sPackageWatchdog = new PackageWatchdog(context);
              }
              return sPackageWatchdog;
          }
      }
  
      /**
       * Called during boot to notify when packages are ready on the device so we can start
       * binding.
       */
      public void onPackagesReady() {
          synchronized (mLock) {
              mIsPackagesReady = true;
              mHealthCheckController.setCallbacks(packageName -> onHealthCheckPassed(packageName),
                      packages -> onSupportedPackages(packages),
                      () -> syncRequestsAsync());
              setPropertyChangedListenerLocked();
              updateConfigs();
              registerNetworkStackHealthListener();
          }
      }
  
      /**
       * Registers {@code observer} to listen for package failures
       *
       * <p>Observers are expected to call this on boot. It does not specify any packages but
       * it will resume observing any packages requested from a previous boot.
       */
      public void registerHealthObserver(PackageHealthObserver observer) {
          synchronized (mLock) {
              ObserverInternal internalObserver = mAllObservers.get(observer.getName());
              if (internalObserver != null) {
                  internalObserver.mRegisteredObserver = observer;
              }
          }
      }
  
      /**
       * Starts observing the health of the {@code packages} for {@code observer} and notifies
       * {@code observer} of any package failures within the monitoring duration.
       *
       * <p>If monitoring a package supporting explicit health check, at the end of the monitoring
       * duration if {@link #onHealthCheckPassed} was never called,
       * {@link PackageHealthObserver#execute} will be called as if the package failed.
       *
       * <p>If {@code observer} is already monitoring a package in {@code packageNames},
       * the monitoring window of that package will be reset to {@code durationMs} and the health
       * check state will be reset to a default depending on if the package is contained in
       * {@link mPackagesWithExplicitHealthCheckEnabled}.
       *
       * @throws IllegalArgumentException if {@code packageNames} is empty
       * or {@code durationMs} is less than 1
       */
      public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames,
              long durationMs) {
          if (packageNames.isEmpty()) {
              Slog.wtf(TAG, "No packages to observe, " + observer.getName());
              return;
          }
          if (durationMs < 1) {
              // TODO: Instead of failing, monitor for default? 48hrs?
              throw new IllegalArgumentException("Invalid duration " + durationMs + "ms for observer "
                      + observer.getName() + ". Not observing packages " + packageNames);
          }
  
          List<MonitoredPackage> packages = new ArrayList<>();
          for (int i = 0; i < packageNames.size(); i++) {
              // Health checks not available yet so health check state will start INACTIVE
              packages.add(new MonitoredPackage(packageNames.get(i), durationMs, false));
          }
  
          // Sync before we add the new packages to the observers. This will #pruneObservers,
          // causing any elapsed time to be deducted from all existing packages before we add new
          // packages. This maintains the invariant that the elapsed time for ALL (new and existing)
          // packages is the same.
          syncState("observing new packages");
  
          synchronized (mLock) {
              ObserverInternal oldObserver = mAllObservers.get(observer.getName());
              if (oldObserver == null) {
                  Slog.d(TAG, observer.getName() + " started monitoring health "
                          + "of packages " + packageNames);
                  mAllObservers.put(observer.getName(),
                          new ObserverInternal(observer.getName(), packages));
              } else {
                  Slog.d(TAG, observer.getName() + " added the following "
                          + "packages to monitor " + packageNames);
                  oldObserver.updatePackagesLocked(packages);
              }
          }
  
          // Register observer in case not already registered
          registerHealthObserver(observer);
  
          // Sync after we add the new packages to the observers. We may have received packges
          // requiring an earlier schedule than we are currently scheduled for.
          syncState("updated observers");
      }
  
      /**
       * Unregisters {@code observer} from listening to package failure.
       * Additionally, this stops observing any packages that may have previously been observed
       * even from a previous boot.
       */
      public void unregisterHealthObserver(PackageHealthObserver observer) {
          synchronized (mLock) {
              mAllObservers.remove(observer.getName());
          }
          syncState("unregistering observer: " + observer.getName());
      }
  
      /**
       * Returns packages observed by {@code observer}
       *
       * @return an empty set if {@code observer} has some packages observerd from a previous boot
       * but has not registered itself in the current boot to receive notifications. Returns null
       * if there are no active packages monitored from any boot.
       */
      @Nullable
      public Set<String> getPackages(PackageHealthObserver observer) {
          synchronized (mLock) {
              for (int i = 0; i < mAllObservers.size(); i++) {
                  if (observer.getName().equals(mAllObservers.keyAt(i))) {
                      if (observer.equals(mAllObservers.valueAt(i).mRegisteredObserver)) {
                          return mAllObservers.valueAt(i).mPackages.keySet();
                      }
                      return Collections.emptySet();
                  }
              }
          }
          return null;
      }
  
      /**
       * Called when a process fails due to a crash, ANR or explicit health check.
       *
       * <p>For each package contained in the process, one registered observer with the least user
       * impact will be notified for mitigation.
       *
       * <p>This method could be called frequently if there is a severe problem on the device.
       */
      public void onPackageFailure(List<VersionedPackage> packages,
              @FailureReasons int failureReason) {
          mLongTaskHandler.post(() -> {
              synchronized (mLock) {
                  if (mAllObservers.isEmpty()) {
                      return;
                  }
  
                  for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
                      VersionedPackage versionedPackage = packages.get(pIndex);
                      // Observer that will receive failure for versionedPackage
                      PackageHealthObserver currentObserverToNotify = null;
                      int currentObserverImpact = Integer.MAX_VALUE;
  
                      // Find observer with least user impact
                      for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
                          ObserverInternal observer = mAllObservers.valueAt(oIndex);
                          PackageHealthObserver registeredObserver = observer.mRegisteredObserver;
                          if (registeredObserver != null
                                  && observer.onPackageFailureLocked(
                                          versionedPackage.getPackageName())) {
                              int impact = registeredObserver.onHealthCheckFailed(versionedPackage);
                              if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
                                      && impact < currentObserverImpact) {
                                  currentObserverToNotify = registeredObserver;
                                  currentObserverImpact = impact;
                              }
                          }
                      }
  
                      // Execute action with least user impact
                      if (currentObserverToNotify != null) {
                          currentObserverToNotify.execute(versionedPackage, failureReason);
                      }
                  }
              }
          });
      }
  
      // TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also
      // avoid holding lock?
      // This currently adds about 7ms extra to shutdown thread
      /** Writes the package information to file during shutdown. */
      public void writeNow() {
          synchronized (mLock) {
              // Must only run synchronous tasks as this runs on the ShutdownThread and no other
              // thread is guaranteed to run during shutdown.
              if (!mAllObservers.isEmpty()) {
                  mLongTaskHandler.removeCallbacks(this::saveToFileAsync);
                  pruneObserversLocked();
                  saveToFile();
                  Slog.i(TAG, "Last write to update package durations");
              }
          }
      }
  
      /**
       * Enables or disables explicit health checks.
       * <p> If explicit health checks are enabled, the health check service is started.
       * <p> If explicit health checks are disabled, pending explicit health check requests are
       * passed and the health check service is stopped.
       */
      private void setExplicitHealthCheckEnabled(boolean enabled) {
          synchronized (mLock) {
              mIsHealthCheckEnabled = enabled;
              mHealthCheckController.setEnabled(enabled);
              // Prune to update internal state whenever health check is enabled/disabled
              syncState("health check state " + (enabled ? "enabled" : "disabled"));
          }
      }
  
      /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
      @Retention(SOURCE)
      @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_NONE,
                       PackageHealthObserverImpact.USER_IMPACT_LOW,
                       PackageHealthObserverImpact.USER_IMPACT_MEDIUM,
                       PackageHealthObserverImpact.USER_IMPACT_HIGH})
      public @interface PackageHealthObserverImpact {
          /** No action to take. */
          int USER_IMPACT_NONE = 0;
          /* Action has low user impact, user of a device will barely notice. */
          int USER_IMPACT_LOW = 1;
          /* Action has medium user impact, user of a device will likely notice. */
          int USER_IMPACT_MEDIUM = 3;
          /* Action has high user impact, a last resort, user of a device will be very frustrated. */
          int USER_IMPACT_HIGH = 5;
      }
  
      /** Register instances of this interface to receive notifications on package failure. */
      public interface PackageHealthObserver {
          /**
           * Called when health check fails for the {@code versionedPackage}.
           *
           * @return any one of {@link PackageHealthObserverImpact} to express the impact
           * to the user on {@link #execute}
           */
          @PackageHealthObserverImpact int onHealthCheckFailed(VersionedPackage versionedPackage);
  
          /**
           * Executes mitigation for {@link #onHealthCheckFailed}.
           *
           * @return {@code true} if action was executed successfully, {@code false} otherwise
           */
          boolean execute(VersionedPackage versionedPackage, @FailureReasons int failureReason);
  
          // TODO(b/120598832): Ensure uniqueness?
          /**
           * Identifier for the observer, should not change across device updates otherwise the
           * watchdog may drop observing packages with the old name.
           */
          String getName();
      }
  
      long getTriggerFailureCount() {
          synchronized (mLock) {
              return mTriggerFailureCount;
          }
      }
  
      /**
       * Serializes and syncs health check requests with the {@link ExplicitHealthCheckController}.
       */
      private void syncRequestsAsync() {
          mShortTaskHandler.removeCallbacks(this::syncRequests);
          mShortTaskHandler.post(this::syncRequests);
      }
  
      /**
       * Syncs health check requests with the {@link ExplicitHealthCheckController}.
       * Calls to this must be serialized.
       *
       * @see #syncRequestsAsync
       */
      private void syncRequests() {
          Set<String> packages = null;
          synchronized (mLock) {
              if (mIsPackagesReady) {
                  packages = getPackagesPendingHealthChecksLocked();
              } // else, we will sync requests when packages become ready
          }
  
          // Call outside lock to avoid holding lock when calling into the controller.
          if (packages != null) {
              Slog.i(TAG, "Syncing health check requests for packages: " + packages);
              mHealthCheckController.syncRequests(packages);
          }
      }
  
      /**
       * Updates the observers monitoring {@code packageName} that explicit health check has passed.
       *
       * <p> This update is strictly for registered observers at the time of the call
       * Observers that register after this signal will have no knowledge of prior signals and will
       * effectively behave as if the explicit health check hasn't passed for {@code packageName}.
       *
       * <p> {@code packageName} can still be considered failed if reported by
       * {@link #onPackageFailureLocked} before the package expires.
       *
       * <p> Triggered by components outside the system server when they are fully functional after an
       * update.
       */
      private void onHealthCheckPassed(String packageName) {
          Slog.i(TAG, "Health check passed for package: " + packageName);
          boolean isStateChanged = false;
  
          synchronized (mLock) {
              for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) {
                  ObserverInternal observer = mAllObservers.valueAt(observerIdx);
                  MonitoredPackage monitoredPackage = observer.mPackages.get(packageName);
  
                  if (monitoredPackage != null) {
                      int oldState = monitoredPackage.getHealthCheckStateLocked();
                      int newState = monitoredPackage.tryPassHealthCheckLocked();
                      isStateChanged |= oldState != newState;
                  }
              }
          }
  
          if (isStateChanged) {
              syncState("health check passed for " + packageName);
          }
      }
  
      private void onSupportedPackages(List<PackageConfig> supportedPackages) {
          boolean isStateChanged = false;
  
          Map<String, Long> supportedPackageTimeouts = new ArrayMap<>();
          Iterator<PackageConfig> it = supportedPackages.iterator();
          while (it.hasNext()) {
              PackageConfig info = it.next();
              supportedPackageTimeouts.put(info.getPackageName(), info.getHealthCheckTimeoutMillis());
          }
  
          synchronized (mLock) {
              Slog.d(TAG, "Received supported packages " + supportedPackages);
              Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
              while (oit.hasNext()) {
                  Iterator<MonitoredPackage> pit = oit.next().mPackages.values().iterator();
                  while (pit.hasNext()) {
                      MonitoredPackage monitoredPackage = pit.next();
                      String packageName = monitoredPackage.getName();
                      int oldState = monitoredPackage.getHealthCheckStateLocked();
                      int newState;
  
                      if (supportedPackageTimeouts.containsKey(packageName)) {
                          // Supported packages become ACTIVE if currently INACTIVE
                          newState = monitoredPackage.setHealthCheckActiveLocked(
                                  supportedPackageTimeouts.get(packageName));
                      } else {
                          // Unsupported packages are marked as PASSED unless already FAILED
                          newState = monitoredPackage.tryPassHealthCheckLocked();
                      }
                      isStateChanged |= oldState != newState;
                  }
              }
          }
  
          if (isStateChanged) {
              syncState("updated health check supported packages " + supportedPackages);
          }
      }
  
      @GuardedBy("mLock")
      private Set<String> getPackagesPendingHealthChecksLocked() {
          Slog.d(TAG, "Getting all observed packages pending health checks");
          Set<String> packages = new ArraySet<>();
          Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
          while (oit.hasNext()) {
              ObserverInternal observer = oit.next();
              Iterator<MonitoredPackage> pit =
                      observer.mPackages.values().iterator();
              while (pit.hasNext()) {
                  MonitoredPackage monitoredPackage = pit.next();
                  String packageName = monitoredPackage.getName();
                  if (monitoredPackage.isPendingHealthChecksLocked()) {
                      packages.add(packageName);
                  }
              }
          }
          return packages;
      }
  
      /**
       * Syncs the state of the observers.
       *
       * <p> Prunes all observers, saves new state to disk, syncs health check requests with the
       * health check service and schedules the next state sync.
       */
      private void syncState(String reason) {
          synchronized (mLock) {
              Slog.i(TAG, "Syncing state, reason: " + reason);
              pruneObserversLocked();
  
              saveToFileAsync();
              syncRequestsAsync();
  
              // Done syncing state, schedule the next state sync
              scheduleNextSyncStateLocked();
          }
      }
  
      private void syncStateWithScheduledReason() {
          syncState("scheduled");
      }
  
      @GuardedBy("mLock")
      private void scheduleNextSyncStateLocked() {
          long durationMs = getNextStateSyncMillisLocked();
          mShortTaskHandler.removeCallbacks(this::syncStateWithScheduledReason);
          if (durationMs == Long.MAX_VALUE) {
              Slog.i(TAG, "Cancelling state sync, nothing to sync");
              mUptimeAtLastStateSync = 0;
          } else {
              Slog.i(TAG, "Scheduling next state sync in " + durationMs + "ms");
              mUptimeAtLastStateSync = SystemClock.uptimeMillis();
              mShortTaskHandler.postDelayed(this::syncStateWithScheduledReason, durationMs);
          }
      }
  
      /**
       * Returns the next duration in millis to sync the watchdog state.
       *
       * @returns Long#MAX_VALUE if there are no observed packages.
       */
      @GuardedBy("mLock")
      private long getNextStateSyncMillisLocked() {
          long shortestDurationMs = Long.MAX_VALUE;
          for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
              ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).mPackages;
              for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
                  MonitoredPackage mp = packages.valueAt(pIndex);
                  long duration = mp.getShortestScheduleDurationMsLocked();
                  if (duration < shortestDurationMs) {
                      shortestDurationMs = duration;
                  }
              }
          }
          return shortestDurationMs;
      }
  
      /**
       * Removes {@code elapsedMs} milliseconds from all durations on monitored packages
       * and updates other internal state.
       */
      @GuardedBy("mLock")
      private void pruneObserversLocked() {
          long elapsedMs = mUptimeAtLastStateSync == 0
                  ? 0 : SystemClock.uptimeMillis() - mUptimeAtLastStateSync;
          if (elapsedMs <= 0) {
              Slog.i(TAG, "Not pruning observers, elapsed time: " + elapsedMs + "ms");
              return;
          }
  
          Slog.i(TAG, "Removing " + elapsedMs + "ms from all packages on all observers");
          Iterator<ObserverInternal> it = mAllObservers.values().iterator();
          while (it.hasNext()) {
              ObserverInternal observer = it.next();
              Set<MonitoredPackage> failedPackages =
                      observer.prunePackagesLocked(elapsedMs);
              if (!failedPackages.isEmpty()) {
                  onHealthCheckFailed(observer, failedPackages);
              }
              if (observer.mPackages.isEmpty()) {
                  Slog.i(TAG, "Discarding observer " + observer.mName + ". All packages expired");
                  it.remove();
              }
          }
      }
  
      private void onHealthCheckFailed(ObserverInternal observer,
              Set<MonitoredPackage> failedPackages) {
          mLongTaskHandler.post(() -> {
              synchronized (mLock) {
                  PackageHealthObserver registeredObserver = observer.mRegisteredObserver;
                  if (registeredObserver != null) {
                      Iterator<MonitoredPackage> it = failedPackages.iterator();
                      while (it.hasNext()) {
                          String failedPackage = it.next().getName();
                          Slog.i(TAG, "Explicit health check failed for package " + failedPackage);
                          VersionedPackage versionedPkg = getVersionedPackage(failedPackage);
                          if (versionedPkg == null) {
                              Slog.w(TAG, "Explicit health check failed but could not find package "
                                      + failedPackage);
                              // TODO(b/120598832): Skip. We only continue to pass tests for now since
                              // the tests don't install any packages
                              versionedPkg = new VersionedPackage(failedPackage, 0L);
                          }
                          registeredObserver.execute(versionedPkg,
                                  PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
                      }
                  }
              }
          });
      }
  
      @Nullable
      private VersionedPackage getVersionedPackage(String packageName) {
          final PackageManager pm = mContext.getPackageManager();
          if (pm == null) {
              return null;
          }
          try {
              final long versionCode = pm.getPackageInfo(
                      packageName, 0 /* flags */).getLongVersionCode();
              return new VersionedPackage(packageName, versionCode);
          } catch (PackageManager.NameNotFoundException e) {
              return null;
          }
      }
  
      /**
       * Loads mAllObservers from file.
       *
       * <p>Note that this is <b>not</b> thread safe and should only called be called
       * from the constructor.
       */
      private void loadFromFile() {
          InputStream infile = null;
          mAllObservers.clear();
          try {
              infile = mPolicyFile.openRead();
              final XmlPullParser parser = Xml.newPullParser();
              parser.setInput(infile, StandardCharsets.UTF_8.name());
              XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG);
              int outerDepth = parser.getDepth();
              while (XmlUtils.nextElementWithin(parser, outerDepth)) {
                  ObserverInternal observer = ObserverInternal.read(parser, this);
                  if (observer != null) {
                      mAllObservers.put(observer.mName, observer);
                  }
              }
          } catch (FileNotFoundException e) {
              // Nothing to monitor
          } catch (IOException | NumberFormatException | XmlPullParserException e) {
              Slog.wtf(TAG, "Unable to read monitored packages, deleting file", e);
              mPolicyFile.delete();
          } finally {
              IoUtils.closeQuietly(infile);
          }
      }
  
      /** Adds a {@link DeviceConfig#OnPropertiesChangedListener}. */
      private void setPropertyChangedListenerLocked() {
          DeviceConfig.addOnPropertiesChangedListener(
                  DeviceConfig.NAMESPACE_ROLLBACK,
                  mContext.getMainExecutor(),
                  (properties) -> {
                      if (!DeviceConfig.NAMESPACE_ROLLBACK.equals(properties.getNamespace())) {
                          return;
                      }
                      updateConfigs();
                  });
      }
  
      /**
       * Health check is enabled or disabled after reading the flags
       * from DeviceConfig.
       */
      private void updateConfigs() {
          synchronized (mLock) {
              mTriggerFailureCount = DeviceConfig.getInt(
                      DeviceConfig.NAMESPACE_ROLLBACK,
                      PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
                      DEFAULT_TRIGGER_FAILURE_COUNT);
              if (mTriggerFailureCount <= 0) {
                  mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
              }
  
              mTriggerFailureDurationMs = DeviceConfig.getInt(
                      DeviceConfig.NAMESPACE_ROLLBACK,
                      PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
                      DEFAULT_TRIGGER_FAILURE_DURATION_MS);
              if (mTriggerFailureDurationMs <= 0) {
                  mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_COUNT;
              }
  
              setExplicitHealthCheckEnabled(DeviceConfig.getBoolean(
                      DeviceConfig.NAMESPACE_ROLLBACK,
                      PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
                      DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED));
          }
      }
  
      private void registerNetworkStackHealthListener() {
          // TODO: have an internal method to trigger a rollback by reporting high severity errors,
          // and rely on ActivityManager to inform the watchdog of severe network stack crashes
          // instead of having this listener in parallel.
          mNetworkStackClient.registerHealthListener(
                  packageName -> {
                      final VersionedPackage pkg = getVersionedPackage(packageName);
                      if (pkg == null) {
                          Slog.wtf(TAG, "NetworkStack failed but could not find its package");
                          return;
                      }
                      // This is a severe failure and recovery should be attempted immediately.
                      // TODO: have a better way to handle such failures.
                      final List<VersionedPackage> pkgList = Collections.singletonList(pkg);
                      final long failureCount = getTriggerFailureCount();
                      for (int i = 0; i < failureCount; i++) {
                          onPackageFailure(pkgList, FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
                      }
                  });
      }
  
      /**
       * Persists mAllObservers to file. Threshold information is ignored.
       */
      private boolean saveToFile() {
          Slog.i(TAG, "Saving observer state to file");
          synchronized (mLock) {
              FileOutputStream stream;
              try {
                  stream = mPolicyFile.startWrite();
              } catch (IOException e) {
                  Slog.w(TAG, "Cannot update monitored packages", e);
                  return false;
              }
  
              try {
                  XmlSerializer out = new FastXmlSerializer();
                  out.setOutput(stream, StandardCharsets.UTF_8.name());
                  out.startDocument(null, true);
                  out.startTag(null, TAG_PACKAGE_WATCHDOG);
                  out.attribute(null, ATTR_VERSION, Integer.toString(DB_VERSION));
                  for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
                      mAllObservers.valueAt(oIndex).writeLocked(out);
                  }
                  out.endTag(null, TAG_PACKAGE_WATCHDOG);
                  out.endDocument();
                  mPolicyFile.finishWrite(stream);
                  return true;
              } catch (IOException e) {
                  Slog.w(TAG, "Failed to save monitored packages, restoring backup", e);
                  mPolicyFile.failWrite(stream);
                  return false;
              } finally {
                  IoUtils.closeQuietly(stream);
              }
          }
      }
  
      private void saveToFileAsync() {
          if (!mLongTaskHandler.hasCallbacks(this::saveToFile)) {
              mLongTaskHandler.post(this::saveToFile);
          }
      }
  
      /**
       * Represents an observer monitoring a set of packages along with the failure thresholds for
       * each package.
       *
       * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
       * instances of this class.
       */
      //TODO(b/120598832): Remove 'm' from non-private fields
      private static class ObserverInternal {
          public final String mName;
          //TODO(b/120598832): Add getter for mPackages
          @GuardedBy("mLock")
          public final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
          @Nullable
          @GuardedBy("mLock")
          public PackageHealthObserver mRegisteredObserver;
  
          ObserverInternal(String name, List<MonitoredPackage> packages) {
              mName = name;
              updatePackagesLocked(packages);
          }
  
          /**
           * Writes important {@link MonitoredPackage} details for this observer to file.
           * Does not persist any package failure thresholds.
           */
          @GuardedBy("mLock")
          public boolean writeLocked(XmlSerializer out) {
              try {
                  out.startTag(null, TAG_OBSERVER);
                  out.attribute(null, ATTR_NAME, mName);
                  for (int i = 0; i < mPackages.size(); i++) {
                      MonitoredPackage p = mPackages.valueAt(i);
                      p.writeLocked(out);
                  }
                  out.endTag(null, TAG_OBSERVER);
                  return true;
              } catch (IOException e) {
                  Slog.w(TAG, "Cannot save observer", e);
                  return false;
              }
          }
  
          @GuardedBy("mLock")
          public void updatePackagesLocked(List<MonitoredPackage> packages) {
              for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
                  MonitoredPackage p = packages.get(pIndex);
                  mPackages.put(p.mName, p);
              }
          }
  
          /**
           * Reduces the monitoring durations of all packages observed by this observer by
           * {@code elapsedMs}. If any duration is less than 0, the package is removed from
           * observation. If any health check duration is less than 0, the health check result
           * is evaluated.
           *
           * @return a {@link Set} of packages that were removed from the observer without explicit
           * health check passing, or an empty list if no package expired for which an explicit health
           * check was still pending
           */
          @GuardedBy("mLock")
          private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) {
              Set<MonitoredPackage> failedPackages = new ArraySet<>();
              Iterator<MonitoredPackage> it = mPackages.values().iterator();
              while (it.hasNext()) {
                  MonitoredPackage p = it.next();
                  int oldState = p.getHealthCheckStateLocked();
                  int newState = p.handleElapsedTimeLocked(elapsedMs);
                  if (oldState != MonitoredPackage.STATE_FAILED
                          && newState == MonitoredPackage.STATE_FAILED) {
                      Slog.i(TAG, "Package " + p.mName + " failed health check");
                      failedPackages.add(p);
                  }
                  if (p.isExpiredLocked()) {
                      it.remove();
                  }
              }
              return failedPackages;
          }
  
          /**
           * Increments failure counts of {@code packageName}.
           * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise
           */
          @GuardedBy("mLock")
          public boolean onPackageFailureLocked(String packageName) {
              MonitoredPackage p = mPackages.get(packageName);
              if (p != null) {
                  return p.onFailureLocked();
              }
              return false;
          }
  
          /**
           * Returns one ObserverInternal from the {@code parser} and advances its state.
           *
           * <p>Note that this method is <b>not</b> thread safe. It should only be called from
           * #loadFromFile which in turn is only called on construction of the
           * singleton PackageWatchdog.
           **/
          public static ObserverInternal read(XmlPullParser parser, PackageWatchdog watchdog) {
              String observerName = null;
              if (TAG_OBSERVER.equals(parser.getName())) {
                  observerName = parser.getAttributeValue(null, ATTR_NAME);
                  if (TextUtils.isEmpty(observerName)) {
                      Slog.wtf(TAG, "Unable to read observer name");
                      return null;
                  }
              }
              List<MonitoredPackage> packages = new ArrayList<>();
              int innerDepth = parser.getDepth();
              try {
                  while (XmlUtils.nextElementWithin(parser, innerDepth)) {
                      if (TAG_PACKAGE.equals(parser.getName())) {
                          try {
                              String packageName = parser.getAttributeValue(null, ATTR_NAME);
                              long duration = Long.parseLong(
                                      parser.getAttributeValue(null, ATTR_DURATION));
                              long healthCheckDuration = Long.parseLong(
                                      parser.getAttributeValue(null,
                                              ATTR_EXPLICIT_HEALTH_CHECK_DURATION));
                              boolean hasPassedHealthCheck = Boolean.parseBoolean(
                                      parser.getAttributeValue(null, ATTR_PASSED_HEALTH_CHECK));
                              if (!TextUtils.isEmpty(packageName)) {
                                  packages.add(watchdog.new MonitoredPackage(packageName, duration,
                                          healthCheckDuration, hasPassedHealthCheck));
                              }
                          } catch (NumberFormatException e) {
                              Slog.wtf(TAG, "Skipping package for observer " + observerName, e);
                              continue;
                          }
                      }
                  }
              } catch (XmlPullParserException | IOException e) {
                  Slog.wtf(TAG, "Unable to read observer " + observerName, e);
                  return null;
              }
              if (packages.isEmpty()) {
                  return null;
              }
              return new ObserverInternal(observerName, packages);
          }
      }
  
      /**
       * Represents a package and its health check state along with the time
       * it should be monitored for.
       *
       * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
       * instances of this class.
       */
      class MonitoredPackage {
          // Health check states
          // TODO(b/120598832): Prefix with HEALTH_CHECK
          // mName has not passed health check but has requested a health check
          public static final int STATE_ACTIVE = 0;
          // mName has not passed health check and has not requested a health check
          public static final int STATE_INACTIVE = 1;
          // mName has passed health check
          public static final int STATE_PASSED = 2;
          // mName has failed health check
          public static final int STATE_FAILED = 3;
  
          //TODO(b/120598832): VersionedPackage?
          private final String mName;
          // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after
          // methods that could change the health check state: handleElapsedTimeLocked and
          // tryPassHealthCheckLocked
          private int mHealthCheckState = STATE_INACTIVE;
          // Whether an explicit health check has passed.
          // This value in addition with mHealthCheckDurationMs determines the health check state
          // of the package, see #getHealthCheckStateLocked
          @GuardedBy("mLock")
          private boolean mHasPassedHealthCheck;
          // System uptime duration to monitor package.
          @GuardedBy("mLock")
          private long mDurationMs;
          // System uptime duration to check the result of an explicit health check
          // Initially, MAX_VALUE until we get a value from the health check service
          // and request health checks.
          // This value in addition with mHasPassedHealthCheck determines the health check state
          // of the package, see #getHealthCheckStateLocked
          @GuardedBy("mLock")
          private long mHealthCheckDurationMs = Long.MAX_VALUE;
          // System uptime of first package failure
          @GuardedBy("mLock")
          private long mUptimeStartMs;
          // Number of failures since mUptimeStartMs
          @GuardedBy("mLock")
          private int mFailures;
  
          MonitoredPackage(String name, long durationMs, boolean hasPassedHealthCheck) {
              this(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck);
          }
  
          MonitoredPackage(String name, long durationMs, long healthCheckDurationMs,
                  boolean hasPassedHealthCheck) {
              mName = name;
              mDurationMs = durationMs;
              mHealthCheckDurationMs = healthCheckDurationMs;
              mHasPassedHealthCheck = hasPassedHealthCheck;
              updateHealthCheckStateLocked();
          }
  
          /** Writes the salient fields to disk using {@code out}. */
          @GuardedBy("mLock")
          public void writeLocked(XmlSerializer out) throws IOException {
              out.startTag(null, TAG_PACKAGE);
              out.attribute(null, ATTR_NAME, mName);
              out.attribute(null, ATTR_DURATION, String.valueOf(mDurationMs));
              out.attribute(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION,
                      String.valueOf(mHealthCheckDurationMs));
              out.attribute(null, ATTR_PASSED_HEALTH_CHECK,
                      String.valueOf(mHasPassedHealthCheck));
              out.endTag(null, TAG_PACKAGE);
          }
  
          /**
           * Increment package failures or resets failure count depending on the last package failure.
           *
           * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise
           */
          @GuardedBy("mLock")
          public boolean onFailureLocked() {
              final long now = SystemClock.uptimeMillis();
              final long duration = now - mUptimeStartMs;
              if (duration > mTriggerFailureDurationMs) {
                  // TODO(b/120598832): Reseting to 1 is not correct
                  // because there may be more than 1 failure in the last trigger window from now
                  // This is the RescueParty impl, will leave for now
                  mFailures = 1;
                  mUptimeStartMs = now;
              } else {
                  mFailures++;
              }
              boolean failed = mFailures >= mTriggerFailureCount;
              if (failed) {
                  mFailures = 0;
              }
              return failed;
          }
  
          /**
           * Sets the initial health check duration.
           *
           * @return the new health check state
           */
          @GuardedBy("mLock")
          public int setHealthCheckActiveLocked(long initialHealthCheckDurationMs) {
              if (initialHealthCheckDurationMs <= 0) {
                  Slog.wtf(TAG, "Cannot set non-positive health check duration "
                          + initialHealthCheckDurationMs + "ms for package " + mName
                          + ". Using total duration " + mDurationMs + "ms instead");
                  initialHealthCheckDurationMs = mDurationMs;
              }
              if (mHealthCheckState == STATE_INACTIVE) {
                  // Transitions to ACTIVE
                  mHealthCheckDurationMs = initialHealthCheckDurationMs;
              }
              return updateHealthCheckStateLocked();
          }
  
          /**
           * Updates the monitoring durations of the package.
           *
           * @return the new health check state
           */
          @GuardedBy("mLock")
          public int handleElapsedTimeLocked(long elapsedMs) {
              if (elapsedMs <= 0) {
                  Slog.w(TAG, "Cannot handle non-positive elapsed time for package " + mName);
                  return mHealthCheckState;
              }
              // Transitions to FAILED if now <= 0 and health check not passed
              mDurationMs -= elapsedMs;
              if (mHealthCheckState == STATE_ACTIVE) {
                  // We only update health check durations if we have #setHealthCheckActiveLocked
                  // This ensures we don't leave the INACTIVE state for an unexpected elapsed time
                  // Transitions to FAILED if now <= 0 and health check not passed
                  mHealthCheckDurationMs -= elapsedMs;
              }
              return updateHealthCheckStateLocked();
          }
  
          /**
           * Marks the health check as passed and transitions to {@link #STATE_PASSED}
           * if not yet {@link #STATE_FAILED}.
           *
           * @return the new health check state
           */
          @GuardedBy("mLock")
          public int tryPassHealthCheckLocked() {
              if (mHealthCheckState != STATE_FAILED) {
                  // FAILED is a final state so only pass if we haven't failed
                  // Transition to PASSED
                  mHasPassedHealthCheck = true;
              }
              return updateHealthCheckStateLocked();
          }
  
          /** Returns the monitored package name. */
          private String getName() {
              return mName;
          }
  
          //TODO(b/120598832): IntDef
          /**
           * Returns the current health check state, any of {@link #STATE_ACTIVE},
           * {@link #STATE_INACTIVE} or {@link #STATE_PASSED}
           */
          @GuardedBy("mLock")
          public int getHealthCheckStateLocked() {
              return mHealthCheckState;
          }
  
          /**
           * Returns the shortest duration before the package should be scheduled for a prune.
           *
           * @return the duration or {@link Long#MAX_VALUE} if the package should not be scheduled
           */
          @GuardedBy("mLock")
          public long getShortestScheduleDurationMsLocked() {
              // Consider health check duration only if #isPendingHealthChecksLocked is true
              return Math.min(toPositive(mDurationMs),
                      isPendingHealthChecksLocked()
                      ? toPositive(mHealthCheckDurationMs) : Long.MAX_VALUE);
          }
  
          /**
           * Returns {@code true} if the total duration left to monitor the package is less than or
           * equal to 0 {@code false} otherwise.
           */
          @GuardedBy("mLock")
          public boolean isExpiredLocked() {
              return mDurationMs <= 0;
          }
  
          /**
           * Returns {@code true} if the package, {@link #getName} is expecting health check results
           * {@code false} otherwise.
           */
          @GuardedBy("mLock")
          public boolean isPendingHealthChecksLocked() {
              return mHealthCheckState == STATE_ACTIVE || mHealthCheckState == STATE_INACTIVE;
          }
  
          /**
           * Updates the health check state based on {@link #mHasPassedHealthCheck}
           * and {@link #mHealthCheckDurationMs}.
           *
           * @return the new health check state
           */
          @GuardedBy("mLock")
          private int updateHealthCheckStateLocked() {
              int oldState = mHealthCheckState;
              if (mHasPassedHealthCheck) {
                  // Set final state first to avoid ambiguity
                  mHealthCheckState = STATE_PASSED;
              } else if (mHealthCheckDurationMs <= 0 || mDurationMs <= 0) {
                  // Set final state first to avoid ambiguity
                  mHealthCheckState = STATE_FAILED;
              } else if (mHealthCheckDurationMs == Long.MAX_VALUE) {
                  mHealthCheckState = STATE_INACTIVE;
              } else {
                  mHealthCheckState = STATE_ACTIVE;
              }
              Slog.i(TAG, "Updated health check state for package " + mName + ": "
                      + toString(oldState) + " -> " + toString(mHealthCheckState));
              return mHealthCheckState;
          }
  
          /** Returns a {@link String} representation of the current health check state. */
          private String toString(int state) {
              switch (state) {
                  case STATE_ACTIVE:
                      return "ACTIVE";
                  case STATE_INACTIVE:
                      return "INACTIVE";
                  case STATE_PASSED:
                      return "PASSED";
                  case STATE_FAILED:
                      return "FAILED";
                  default:
                      return "UNKNOWN";
              }
          }
  
          /** Returns {@code value} if it is greater than 0 or {@link Long#MAX_VALUE} otherwise. */
          private long toPositive(long value) {
              return value > 0 ? value : Long.MAX_VALUE;
          }
      }
  }
  