\section{Introduction}

Different aspects of quantum state discrimination are discussed in the
literature ever since the seminal papers of Helstrom and Holevo
\cite{1,2,3,4,5} and are now presented in many textbooks and reviews, see, for
example, in \cite{6,7,8} and references therein.

Let a sender prepare a quantum system described in terms of a complex Hilbert
space $\mathcal{H}$ in one of $r\geq2$ quantum states $\rho_{1},...,\rho_{r},$
pure or mixed, with probabilities $q_{1},...,q_{r},$ $\sum_{i}q_{i
=1,\mathit{\ }q_{i}>0$, and send this quantum system in an initial state
\begin{equation}
\rho=\sum_{i=1,...,r}q_{i}\rho_{i},\mathit{\ \ \ }\sum_{i=1,..,r
q_{i}=1,\mathit{\ \ \ }q_{i}>0, \label{1
\end{equation}
to a receiver. For discriminating between states $\rho_{1},...,\rho_{r},$ a
receiver performs a measurement described by a POV measure
\begin{equation}
\mathrm{M}_{r}=\left\{  \mathrm{M}_{r}(i),\text{ \ }i=1,...,r;\text{ \ \
\sum_{i=1,...,r}\mathrm{M}_{r}(i)=\mathbb{I}_{\mathcal{H}}\right\}  \label{2
\end{equation}
and the success probability to take under this measurement the proper decision
equals t
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r})=\sum_{i=1,...,r}q_{i}\mathrm{tr}\left\{  \rho_{i
\mathrm{M}_{r}(i)\right\}  , \label{3
\end{equation}
correspondingly, the error probability
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{error
(\mathrm{M}_{r})  &  =\sum_{i=1,...,r}q_{i}\mathrm{tr}\left\{  \rho
_{i}(\mathbb{I}-\mathrm{M}_{r}(i))\right\} \label{3.1}\\
&  =1-\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r}).\nonumber
\end{align}

Denote by $\mathfrak{M}_{r}=\left\{  \mathrm{M}_{r}\right\}  ,$ $r\geq2,$ the
set of all possible POV measures (\ref{2}). Under the maximum likelihood (the
minimum error) state discrimination strategy, the optimal success probability
and the optimal error probability are given b
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{opt.success}  &
:\text{ }=\max_{\mathrm{M}_{r}\text{ }\in\text{ }\mathfrak{M}_{r}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r}),\label{4}\\
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{opt.error}  &
:\text{ }=\min_{\mathrm{M}_{r}\text{ }\in\text{ }\mathfrak{M}_{r}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{error
(\mathrm{M}_{r})=1-\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}
}^{opt.success}, \label{4.1
\end{align}
and are attained at some extreme point of the convex set $\mathfrak{M}_{r}.$

The alternative expressions for the optimal error probability (\ref{4.1}) are
presented in Theorem 1 and Corollary 1 of \cite{9-1}.

The following general statement was first formulated and proved by Holevo in
\cite{3,4}.

\begin{theorem}
Under the maximum likelihood (the minimum error) state discrimination
strategy, a POV measure $\mathrm{M}_{r}^{(opt)}$ $\in$ $\mathfrak{M}_{r}$ is
optimal if and only if there exists a self-adjoint trace class operator
$\Lambda_{0}$ such that: \textrm{(i)} $\left(  \Lambda_{0}-q_{i}\rho
_{i}\right)  \mathrm{M}_{r}^{(opt)}(i)=0;$ \textrm{(ii)} $\Lambda_{0}\geq
q_{i}\rho_{i},$ for all $i=1,...,r.$ Herewith
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{opt.success
=\mathrm{tr}\{\Lambda_{0}\},\text{ \ \ \ \ }\Lambda_{0}=\sum_{i=1,...,r
q_{i}\rho_{i}\mathrm{M}_{r}^{(opt)}(i). \label{5.1
\end{equation}

\end{theorem}

For $r=2,$ the success probability (\ref{3}) admits the Helstrom upper bound
\cite{1,2,5}
\begin{equation}
\mathrm{P}_{\rho_{1},\rho_{2}|q_{1},q_{2}}^{success}(\mathrm{M}_{2})\leq
\frac{1}{2}\left(  1+\left\Vert q_{1}\rho_{1}-q_{2}\rho_{2}\right\Vert
_{1}\right)  , \label{5
\end{equation}
which is attained, so that, for $r=2,$ the optimal success probability is
given by \cite{1,2,5}
\begin{equation}
\mathrm{P}_{\rho_{1},\rho_{2}|q_{1},q_{2}}^{opt.success}=\frac{1}{2}\left(
1+\left\Vert q_{1}\rho_{1}-q_{2}\rho_{2}\right\Vert _{1}\right)  , \label{6
\end{equation}
where $\left\Vert \cdot\right\Vert _{1}$ is the trace norm.

For an arbitrary $r>2$, a precise general expression for the optimal success
probability (\ref{4}) is not known, however, there were introduced and studied
\cite{10,11,12,13,14,15,16,17,18, 19,20} several general upper and lower
bounds on the optimal success probability $\mathrm{P}_{\rho_{1},...,\rho
_{_{r}}|q_{1},...,q_{_{r}}}^{opt.success}$, expressed via different
characteristics of quantum states.

As proved by Qiu\&Li \cite{16}, in some cases, the general lower bound on the
optimal error probability (correspondingly, the upper bound on the optimal
success probability $\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}
}^{opt.success}),$ introduced by them in \cite{16}, is tighter than the other
general lower bounds known in the literature.

Computation of bounds on $\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1
,...,q_{_{r}}}^{opt.success}$within semidefinite programming was considered
recently in \cite{20}.

In the present article, for the optimal success probability (\ref{4}), we find
(Theorems 2, 3) for all $r\geq2$ the new general lower and upper bounds and
specify (Propositions 1, 2) the relation of our bounds to the general lower
and upper bounds known in literature. For $r=2$, each of the new general
bounds, lower and upper, reduces to the Helstrom bound in (\ref{5}), and this
proves in the other way the Helstrom result (\ref{6}).

\section{New lower bounds}

Taking into account that $\mathrm{M}_{r}(j)=\mathbb{I}_{\mathcal{H}
-\sum_{i\neq j}\mathrm{M}_{r}(i),$ we rewrite the right-hand side of
expression (\ref{3}) in either of $j$-th representations
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r})  &  =\sum_{i=1,...,r}q_{i}\mathrm{tr}\{\rho_{i}\mathrm{M
_{r}(i)\}\label{8}\\
&  =q_{j}+\sum_{i=1,...,r}\mathrm{tr}\{\left(  q_{i}\rho_{i}-q_{j}\rho
_{j}\right)  \mathrm{M}_{r}(i)\},\text{ \ \ }j=1,...,r,\nonumber
\end{align}
for every POV measure $\mathrm{M}_{r}.$ Summing up the left-hand and the
right-hand sides of (\ref{8}) over all $j=1,...,r$, for any POV measure
$\mathrm{M}_{r}\in\mathfrak{M}_{r},$ we also come to the following
representation for the success probabilit
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r})=\frac{1}{r}\left(  1+\sum_{i,j=1,...,r}\mathrm{tr}\left\{
\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  \mathrm{M}_{r}(i)\text{ }\right\}
\right)  . \label{9
\end{equation}

Recall that a self-adjoint (Hermitian) bounded operator $X$ on $\mathcal{H}$
admits the decomposition
\begin{align}
X  &  =X^{(+)}-X^{(-)},\text{ \ \ }X^{(\pm)}\geq0,\label{10}\\
X^{(+)}  &  =\sum_{\lambda_{k}>0,}\lambda_{k}\mathrm{E}_{X}(\lambda
_{k}),\text{ \ \ }X^{(-)}=\sum_{\lambda_{k}\leq0,}\left\vert \lambda
_{k}\right\vert \mathrm{E}_{X}(\lambda_{k}),\nonumber
\end{align}
where $\mathrm{E}_{X}(\lambda_{k})$ the spectral projections of a Hermitian
operator $X$. If a bounded operator $X$ is trace class, then operators
$X^{(\pm)}\geq0$ are also trace class and
\begin{align}
\left\Vert X\right\Vert _{1}  &  :=\mathrm{tr}\left\vert X\right\vert ,\text{
\ \ }\left\vert X\right\vert =X^{(+)}+X^{(-)},\label{11}\\
\left\Vert X^{(\pm)}\right\Vert _{1}  &  =\mathrm{tr}\{X^{(\pm)}\}.\nonumber
\end{align}
From relations \cite{7} $\left\vert \mathrm{tr}\{W\}\right\vert \leq\left\Vert
W\right\Vert _{1}$ and $\left\Vert AB\right\Vert _{1}\leq\left\Vert
A\right\Vert _{1}\left\Vert B\right\Vert _{0}$, valid for all trace-class
operators $W,$ $A$ and all bounded operators $B,$ it follows that if
$X,Y$\ $\geq0$ (hence, $\mathrm{tr}\{XY\}\geq0),$ the
\begin{equation}
0\leq\mathrm{tr}\{XY\}\leq\left\Vert X\right\Vert _{1}\left\Vert Y\right\Vert
_{0}, \label{12
\end{equation}
where notation $\left\Vert \cdot\right\Vert _{0}$ means the operator norm.

Definition (\ref{4}) and\ relations (\ref{8})--(\ref{12}) imply.

\begin{theorem}
[New lower bounds]For any number $r\geq2$ of arbitrary quantum states
$\rho_{1},...,\rho_{r}$ prepared with probabilities $q_{1},...,q_{r}$, the
optimal success probability (\ref{4}) admits the lower bound
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}  &
\geq\mathfrak{L}_{1,new}^{(r)}:=\max_{j=1,...,r}\left\{  q_{j}+\frac{1
{r-1}\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)
^{(+)}\right\Vert _{1}\right\} \label{15}\\
&  =\frac{1}{2(r-1)}+\frac{1}{2(r-1)}\max_{j=1,...,r}\left\{  \sum
_{i=1,...,r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1
+q_{j}(r-2)\right\} \label{16}\\
&  \geq\mathfrak{L}_{2,new}^{(r)}:=\frac{1}{r}\left(  1+\frac{1}{r-1
\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert
_{1}\right)  . \label{17
\end{align}
For $r=2,$ each of these new lower bounds reduces to the Helstrom bound in
(\ref{5}).
\end{theorem}

\begin{proof}
Let $\mathrm{E}_{(q_{i}\rho_{i}-q_{j}\rho_{j})}(\lambda_{k})$ be the spectral
projections of the Hermitian operator $(q_{i}\rho_{i}-q_{j}\rho_{j})$ on
$\mathcal{H}$ and
\begin{equation}
\mathrm{P}_{ij}^{(+)}:=\sum_{\lambda_{k}>0}\mathrm{E}_{(q_{i}\rho_{i
-q_{j}\rho_{j})}(\lambda_{k}),\text{ \ \ }i\neq j, \label{18
\end{equation}
denote the orthogonal projection on the proper subspace of operator
$(q_{i}\rho_{i}-q_{j}\rho_{j}),$ corresponding to its positive eigenvalues.
Note that by (\ref{11})
\begin{align}
q_{i}-q_{j}  &  =\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)
^{(+)}\right\Vert _{1}-\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)
^{(-)}\right\Vert _{1},\label{20.1}\\
\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}  &  =\left\Vert \left(
q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}+\left\Vert \left(
q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(-)}\right\Vert _{1}.\nonumber
\end{align}
Introduce the POV measures $\mathrm{M}_{r}^{(j)},$ $j=1,....,r,$ each with the
element
\begin{align}
\mathrm{M}_{r}^{(j)}(i)  &  =\frac{1}{r-1}\mathrm{P}_{ij}^{(+)},\text{
\ \ }\ i\neq j.\label{19}\\
\mathrm{M}_{r}^{(j)}(j)  &  =\mathbb{I}_{\mathcal{H}}-\frac{1}{r-1
\sum_{i=1,...,r,\text{ }i\neq j}\mathrm{P}_{ij}^{(+)}\nonumber\\
&  =\frac{1}{r-1}\sum_{i=1,...,r,\text{ }i\neq j}(\mathbb{I}_{\mathcal{H
}-\mathrm{P}_{ij}^{(+)})\geq0.\nonumber
\end{align}
From the $j$-th representation in (\ref{8}) and relations (\ref{20.1}) it
follows that, for the $j$-th POV measure (\ref{19}), we have
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{success}(\mathrm{M
_{r}^{(j)})  &  =q_{j}+\sum_{i=1,...,N}\mathrm{tr}\{\left(  q_{i}\rho
_{i}-q_{j}\rho_{j}\right)  \mathrm{M}_{r}^{(j)}(i)\}\label{20}\\
&  =q_{j}+\frac{1}{r-1}\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i
-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1},\nonumber\\
j  &  =1,...,r.\nonumber
\end{align}
For the optimal success probability (\ref{4}), equalities (\ref{20}) impl
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}  &  \geq
q_{j}+\frac{1}{r-1}\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i}-q_{j
\rho_{j}\right)  ^{(+)}\right\Vert _{1},\label{21}\\
\forall j  &  =1,...,r,\nonumber
\end{align}
hence,
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}\geq
\max_{j=1,...,r}\left\{  q_{j}+\frac{1}{r-1}\sum_{i=1,...,r}\left\Vert \left(
q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\right\}  .
\label{22
\end{equation}
Since by (\ref{20.1})
\begin{equation}
\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert
_{1}=\frac{1}{2}(q_{i}-q_{j})+\frac{1}{2}\left\Vert q_{i}\rho_{i}-q_{j
\rho_{j}\right\Vert _{1}, \label{23
\end{equation}
the expression in the right-hand side of (\ref{21}) is otherwise equal t
\begin{align}
&  q_{j}+\frac{1}{r-1}\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i
-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\label{24}\\
&  =\frac{1}{2(r-1)}+\frac{1}{2(r-1)}\left\{  \sum_{i=1,...,r}\left\Vert
q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}+q_{j}(r-2)\right\}  .\nonumber
\end{align}
This and relation (\ref{22}) imply the lower bounds (\ref{15}) and (\ref{16}).
Summing up the left-hand and the right-hand sides of (\ref{21}) over all
$j=1,...,r$ and taking into account $\sum_{j=1,...,r}q_{j}=1,$ relations
(\ref{20.1}) and
\begin{equation}
\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{+}=\left(  q_{j}\rho_{j
-q_{i}\rho_{i}\right)  ^{(-)}, \label{25
\end{equation}
we deriv
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}\geq\frac
{1}{r}\left(  1+\frac{1}{r-1}\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho
_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  , \label{26
\end{equation}
that is, the lower bound (\ref{17}). Furthermore, since, for any positive
numbers $\alpha_{j},$ $j=1,...,r,$ their sum
\begin{equation}
\sum_{j=1,...,r}\alpha_{j}\leq r\max_{j=1,...,r}\alpha_{j}, \label{27
\end{equation}
we hav
\begin{align}
&  \max_{j=1,...,r}\left\{  q_{j}+\frac{1}{r-1}\sum_{i=1,...,N}\left\Vert
\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\right\}
\label{28}\\
&  \geq\frac{1}{r}\left(  1+\frac{1}{r-1}\sum_{1\leq i<j\leq r}\left\Vert
q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  .\nonumber
\end{align}
Relations (\ref{22}), (\ref{24}) and (\ref{28}) prove the statement of Theorem 2.
\end{proof}

Consider now the relation of the new lower bounds (\ref{15})--(\ref{17}) to
the known general lower bounds on $\mathrm{P}_{\rho_{1},...,\rho_{r
|q_{1},...,q_{r}}^{opt.success}:
\begin{align}
\mathfrak{L}_{1}^{(r)}  &  :=\max_{j=1,...,r}q_{j}\geq\frac{1}{r
,\label{28.1}\\
\mathfrak{L}_{2}^{(r)}  &  :=1-\sum_{1\leq i<j\leq r}\sqrt{q_{i}q_{j}
F_{ij},\label{28.2}\\
\mathfrak{L}_{3}^{(r)}  &  :=\left(  \mathrm{tr}\left[  \sqrt
{\textstyle\sum\limits_{i=1,...,r}}
q_{i}^{2}\rho_{i}^{2}}\right]  \right)  ^{2}, \label{28.3
\end{align}
where (a) bound (\ref{28.1}) follows from item (ii) and relation (\ref{5.1})
in Theorem 1; (b) bound (\ref{28.2}) was introduced by Barnum\&Knill in
\cite{9} and further studied by Audenaert\&Mosonyi in \cite{19}; (c) bound
(\ref{28.3}) was introduced by Tyson in \cite{18}. Here, $F_{ij}:=\left\Vert
\sqrt{\rho_{i}}\sqrt{\rho_{j}}\right\Vert _{1}$ is the pairwise fidelity.

\begin{proposition}
(i) The new lower bound (\ref{15}) is tighte
\begin{equation}
\mathfrak{L}_{1,new}^{(r)}\geq\mathfrak{L}_{1}^{(r)} \label{29.1
\end{equation}
than the known lower bound (\ref{28.1}) for any number $r\geq2$ of arbitrary
states $\rho_{1},...,\rho_{r}$ and probabilities $q_{1},...,q_{r}$.
\newline(ii) For $r=2,$ the new lower bounds (\ref{15}), (\ref{17}) are equal
to the Helstrom bound and $\mathfrak{L}_{1,new}^{(2)}=\mathfrak{L
_{2,new}^{(2)}\geq\mathfrak{L}_{2}^{(2)}$ for all states $\rho_{1},\rho_{2}$
and probabilities $q_{1},q_{2}.$ \newline(iii) For any $r>2,$ the new lower
bound (\ref{17}) is tighter than the lower bound (\ref{28.2}
\begin{equation}
\mathfrak{L}_{2,new}^{(r)}\geq\mathfrak{L}_{2}^{(r)} \label{29.3
\end{equation}
i
\begin{equation}
\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert
_{1}\leq\frac{(r-1)^{2}}{r+1} \label{29.4
\end{equation}
and otherwise if\
\begin{equation}
\frac{(r-1)^{2}}{r+1}<\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j
\rho_{j}\right\Vert _{1}\leq r-1. \label{29.5
\end{equation}

\end{proposition}

\begin{proof}
Due to the structure of the new lower bound (\ref{15}), relation (\ref{29.1})
is obvious. In order to prove (ii) and (iii), consider the differenc
\begin{equation}
\mathfrak{L}_{2,new}^{(r)}-\mathfrak{L}_{2}^{(r)}=\frac{1}{r}+\frac{1
{r(r-1)}\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho
_{j}\right\Vert _{1}-1+\sum_{1\leq i<j\leq r}\sqrt{q_{i}q_{j}}F(\rho_{i
,\rho_{j}). \label{29.6
\end{equation}
By Lemma 5 in \cite{16}
\begin{equation}
\sqrt{q_{i}q_{j}}F(\rho_{i},\rho_{j})\geq\frac{1}{2}(q_{i}+q_{j})-\frac{1
{2}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}. \label{29.7
\end{equation}
Substituting this relation into (\ref{29.6}) and taking into account
\begin{equation}
\sum_{1\leq i<j\leq r}(q_{i}+q_{j})=r-1, \label{29.8
\end{equation}
we deriv
\begin{align}
\mathfrak{L}_{2,new}^{(r)}-\mathfrak{L}_{2}^{(r)}  &  \geq\frac{(r-2)(r-1)
{2r}-\left(  \frac{1}{2}-\frac{1}{r(r-1)}\right)  \sum_{1\leq i<j\leq
r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\label{29.9}\\
&  =\frac{(r-2)(r-1)}{2r}\left(  1-\frac{r+1}{(r-1)^{2}}\sum_{1\leq i<j\leq
r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  .\nonumber
\end{align}
This proves relations (\ref{29.3}), (\ref{29.4}), also, the left-hand side
inequality of (\ref{29.5}). The right-hand side inequality of (\ref{29.5})
follows from the relatio
\begin{equation}
\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert
_{1}\leq\sum_{1\leq i<j\leq r}(q_{i}+q_{j})=r-1, \label{29.10
\end{equation}
where we took into account (\ref{29.8}).
\end{proof}

\section{New upper bound}

From relations (\ref{10})--(\ref{12}) and inequality $\left\Vert
\mathrm{M}_{r}(i)\right\Vert _{0}\leq1$ it follows that, for every POV measure
$\mathrm{M}_{r},$ in representation (\ref{8}
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r})  &  =q_{k}+\sum_{i=1,...,r}\mathrm{tr}\{(q_{i}\rho_{i
-q_{k}\rho_{k})\mathrm{M}_{r}(i)\}\label{13}\\
&  \leq q_{m}+\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i}-q_{m}\rho
_{m}\right)  ^{(+)}\right\Vert _{1},\text{ \ \ \ }k,m=1,...,r,\nonumber
\end{align}
and in representation (\ref{9})
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{_{r}}}|q_{1},...,q_{_{r}}}^{success
(\mathrm{M}_{r})  &  =\frac{1}{r}\left(  1+\sum_{i,j=1,.,,,r}\mathrm{tr
\left\{  \text{ }\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  \mathrm{M
_{r}(i)\right\}  \right) \label{14}\\
&  \leq\frac{1}{r}\left(  1+\sum_{i,j=1,...,r}\left\Vert \left(  q_{i}\rho
_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\right)  .\nonumber
\end{align}

Relation (\ref{13}) immediately implies the upper boun
\begin{align}
\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{opt.success}  &
\leq Q_{4}^{(r)}:=\min_{j=1,...,r}\left\{  q_{j}+\sum_{i=1,...,r}\left\Vert
\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\right\}
\label{30}\\
&  =\frac{1}{2}+\frac{1}{2}\min_{j=1,...,r}\left\{  \sum_{i=1,...,r}\left\Vert
q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}-q_{j}(r-2)\right\}  ,\nonumber
\end{align}
which agrees due to the relation $\mathrm{P}_{\rho_{1},...,\rho_{_{r}
|q_{1},...,q_{_{r}}}^{opt.success}=1-$ $\mathrm{P}_{\rho_{1},...,\rho_{_{r
}|q_{1},...,q_{_{r}}}^{opt.error}$ with the lower bound $L_{4}$ by Qiu\&Li on
$\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}}}^{opt.error}$
introduced in \cite{16}. Here, in order to derive the expression in the second
line of (\ref{30}) we took into account relation (\ref{23}).

For convenience in comparison, we take for the upper bound (\ref{30}) and the
below upper bounds (\ref{40} )-(\ref{42}) on the optimal success probability
the numeration similar to that for the lower bounds $L_{n}^{(r)}$ in \cite{16}
on the optimal error probability with the obvious correspondence $Q_{n
^{(r)}=1-L_{n}^{(r)}.\medskip$

The following theorem introduces a new upper bound on the optimal success
probability $\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|q_{1},...,q_{_{r}
}^{opt.success}$and establishes its relation to the upper bound (\ref{30}).

\begin{theorem}
[New upper bound]For any number $r\geq2$ of arbitrary quantum states $\rho
_{1},...,\rho_{r_{r}}$ prepared with probabilities $q_{1},...,q_{_{r}},$ the
optimal success probability (\ref{4}) admits the upper boun
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}\leq
Q_{new}^{(r)}:=\frac{1}{r}\left(  1+\sum_{1\leq i<j\leq r}\left\Vert q_{i
\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  , \label{32
\end{equation}
explicitly generalizing to $r>2$ the form of the Helstrom upper bound in
(\ref{5}) and relating to the upper bound (\ref{30}) a
\begin{equation}
Q_{4}^{(r)}\leq Q_{new}^{(r)}. \label{33
\end{equation}

\end{theorem}

\begin{proof}
In view of (\ref{4}), relations (\ref{14}) and (\ref{25}) immediately imply
the upper bound (\ref{32}). Also, by (\ref{13})
\begin{equation}
\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}\leq
q_{j}+\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)
^{(+)}\right\Vert _{1},\text{ \ \ \ }j=1,...,r. \label{34
\end{equation}
Summing up the left-hand and the right-hand sides of this relation over
$j=1,...,r$, and taking into account (\ref{25}), we deriv
\begin{align}
r\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success}  &  \leq
\sum_{j=1,...,r}q_{j}+\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i
-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\label{35}\\
&  =1+\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert
_{1}.\nonumber
\end{align}
Since, for any positive numbers $\alpha_{j},$ $j=1,...,r,$ their sum
\begin{equation}
\sum_{j=1,...,r}\alpha_{j}\geq r\min_{j=1,...,r}\alpha_{j}, \label{36
\end{equation}
we have
\begin{align}
&  \min_{j=1,...,r}\left\{  q_{j}+\sum_{i}\left\Vert \left(  q_{i}\rho
_{i}-q_{j}\rho_{j}\right)  ^{(+)}\right\Vert _{1}\right\} \label{37}\\
&  \leq\frac{1}{r}\left(  1+\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho
_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  ,\nonumber
\end{align}
that is, relation (\ref{33}). This proves the statement of Theorem 3.
\end{proof}

We stress that if $r$ is rather large, then the calculation of bound
(\ref{32}) is easier than finding the maximum in bound (\ref{30}).

\begin{remark}
From relations (\ref{23}) and $\left\Vert q_{i}\rho_{i}-q_{j}\rho
_{j}\right\Vert _{1}\leq q_{i}+q_{j},$ it follows
$\vert
$\vert
$\left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)  ^{(+)}||_{1}\leq q_{i},$
therefore
\begin{equation}
\sum_{i=1,...,r}\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho_{j}\right)
^{(+)}\right\Vert _{1}\leq\sum_{i=1,...,r,\text{ }i\neq j}q_{i}=1-q_{j}.
\label{38_
\end{equation}
Relations (\ref{25}), (\ref{29.8}) and (\ref{38_}) imply
\begin{equation}
\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert
_{1}=\sum_{i,j=1,...,r,\text{ }}\left\Vert \left(  q_{i}\rho_{i}-q_{j}\rho
_{j}\right)  ^{(+)}\right\Vert _{1}\leq r-1, \label{38
\end{equation}
which agrees with (\ref{29.10}). Therefore, the new upper bound in Theorem 3
is nontrivial (i. e. not more than one). Also, by relation (\ref{13})
specified for the POV measure (\ref{19}), the bounds in Theorem 2 and the
upper bound (\ref{30}) are consistent in the sense
\begin{align}
&  \max_{k=1,...,r}\left\{  q_{k}+\frac{1}{r-1}\sum_{i=1,...,r}\left\Vert
\left(  q_{i}\rho_{i}-q_{k}\rho_{k}\right)  ^{(+)}\right\Vert _{1}\right\}
\label{39}\\
&  \leq\min_{m=1,...,r}\left\{  q_{m}+\sum_{i=1,...,r}\left\Vert \left(
q_{i}\rho_{i}-q_{m}\rho_{m}\right)  ^{(+)}\right\Vert _{1}\right\} \nonumber
\end{align}
for all $r\geq2.$
\end{remark}

Besides relation (\ref{33}) of the new upper bound (\ref{32}) to the upper
bound (\ref{30}) introduced in \cite{16}, consider also its relation to the
general upper bounds on $\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r
}^{opt.success}$
\begin{align}
Q_{2}^{(r)}  &  :=\frac{1}{2}\left(  1+\frac{1}{r-1}\sum_{1\leq i<j\leq
r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  ,\label{40}\\
Q_{3}^{(r)}  &  :=1-\sum_{1\leq i<j\leq r}q_{i}q_{j}F_{ij}^{2},\label{41}\\
Q_{5}^{(r)}  &  :=\mathrm{tr}\left[  \sqrt
{\textstyle\sum\limits_{i=1,...,r}}
q_{i}^{2}\rho_{i}^{2}}\right]  , \label{42
\end{align}
which follow from the lower bounds $L_{n}^{(r)}$ on the optimal error
probability introduced, correspondingly: (i) by Qiu in \cite{14}; (ii) by
Montanaro in \cite{15}; (iii) by Ogawa\&Nagaoka in \cite{17} for the
equiprobable case and by Tyson \cite{18} for a general case.

The detailed study of these general bounds and some other bounds for specific
families of quantum states is presented in \cite{19}.

\begin{proposition}
(i) For any number $r\geq2,$ of arbitrary quantum states $\rho_{1
,...,\rho_{r}$ and a priori probabilities $q_{1},...,q_{r}$, the new upper
bound (\ref{32}) and the upper bounds (\ref{40})--(\ref{41}) satisfy the
relation
\begin{equation}
Q_{new}^{(r)}\leq Q_{2}^{(r)}\leq1-\frac{1}{N-1}(1-Q_{3}^{(r)}). \label{44
\end{equation}
\newline(ii) In the equiprobable case, the new upper bound (\ref{32}) is
tighter than the known upper bound (\ref{41}):
\begin{equation}
Q_{new}^{(r)}\leq Q_{3}^{(r)}. \label{44.1
\end{equation}

\end{proposition}

\begin{proof}
(i) The first inequality in (\ref{44}) follows from
\begin{align}
&  Q_{2}^{(r)}-Q_{new}^{(r)}\label{46}\\
&  =\frac{r-2}{2r}\left(  1-\frac{1}{r-1}\sum_{1\leq i<j\leq r}\left\Vert
q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  \geq0,\nonumber
\end{align}
where we take into account estimate (\ref{38}). To prove the second inequality
in (\ref{44}), we use estimate (55) in \cite{15} and deriv
\begin{equation}
1-Q_{2}^{(r)}\geq\frac{1-Q_{3}^{(r)}}{r-1}. \label{47
\end{equation}
(ii) For the equiprobable case $q_{1}=...=q_{r}=\frac{1}{r}$, we consider the differenc

\begin{align}
Q_{new}^{(r)}-Q_{3}^{(r)}  &  =\frac{1}{r}+\frac{1}{r^{2}}\sum_{1\leq i<j\leq
r}\left\Vert \rho_{i}-\rho_{j}\right\Vert _{1}-1+\frac{1}{r^{2}}\sum_{1\leq
i<j\leq r}F^{2}(\rho_{i},\rho_{j})\label{47.1}\\
&  =\frac{1}{r^{2}}\sum_{1\leq i<j\leq r}\left(  F^{2}(\rho_{i},\rho
_{j})+\left\Vert \rho_{i}-\rho_{j}\right\Vert _{1}\right)  -\frac{r-1
{r}.\nonumber
\end{align}
Taking into account relation (20) in \cite{16} which implie
\begin{equation}
\sum_{1\leq i<j\leq r}\left(  F^{2}(\rho_{i},\rho_{j})+\left\Vert \rho
_{i}-\rho_{j}\right\Vert _{1}\right)  \leq\sum_{1\leq i<j\leq r}2=r(r-1)
\label{47.2
\end{equation}
and substituting this into (\ref{47.1}), we hav
\begin{equation}
Q_{new}^{(r)}-Q_{3}^{(r)}\leq\frac{r-1}{r}-\frac{r-1}{r}=0. \label{47.3
\end{equation}
This and relation (\ref{46}) prove the statement.
\end{proof}

\section{General relations}

Theorem 2 and Theorem 3 imply. the following general relations.

\begin{corollary}
For any number $r\geq2$ of arbitrary quantum states $\rho_{1},...,\rho_{_{r}}$
prepared with any probabilities $q_{1},...,q_{_{r}},$ the optimal success
probability (\ref{4}) admits the bounds
\begin{align}
&  \frac{1}{r}\left(  1+\frac{1}{(r-1)}\sum_{1\leq i<j\leq r}\left\Vert
q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}\right) \nonumber\\
&  \leq\frac{1}{2(r-1)}+\frac{1}{2(r-1)}\max_{j=1,...,r}\left\{
\sum_{i=1,...,r}\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1
+q_{j}(r-2)\right\} \nonumber\\
&  \leq\mathrm{P}_{\rho_{1},...,\rho_{r}|q_{1},...,q_{r}}^{opt.success
\label{49}\\
&  \leq\frac{1}{2}+\frac{1}{2}\min_{j=1,...,r}\left\{  \sum_{i=1,...,r
\left\Vert q_{i}\rho_{i}-q_{j}\rho_{j}\right\Vert _{1}-q_{j}(r-2)\right\}
\nonumber\\
&  \leq\frac{1}{r}\left(  1+\sum_{1\leq i<j\leq r}\left\Vert q_{i}\rho
_{i}-q_{j}\rho_{j}\right\Vert _{1}\right)  .\nonumber
\end{align}
In the first and the last lines of (\ref{49}), the bounds are quite similar by
its form to the Helstrom upper bound (\ref{5}). For $r=2$, each of the lower
and all bounds in (\ref{49}) reduces to the Helstrom bound in (\ref{5}) and
this proves in the other way the Helstrom result (\ref{6}).
\end{corollary}

For the equiprobable case, bounds (\ref{49}) take the forms.

\begin{corollary}
For any number $r\geq2,$ of arbitrary quantum states $\rho_{1},...,\rho_{r},$
prepared with equal probabilities $q_{1}=...=q_{1}=\frac{1}{r},$ the optimal
success probability (\ref{4}) admits the bounds
\begin{align}
&  \frac{1}{r}\left(  1+\frac{1}{r(r-1)}\sum_{1\leq i<j\leq r}\left\Vert
\rho_{i}-\rho_{j}\right\Vert _{1}\right) \nonumber\\
&  \leq\frac{1}{r}+\frac{1}{2r(r-1)}\max_{j=1,...,r}\left\{  \sum
_{i=1,...,r}\left\Vert \rho_{i}-\rho_{j}\right\Vert _{1}\right\} \nonumber\\
&  \leq\mathrm{P}_{\rho_{1},...,\rho_{_{r}}|\frac{1}{r},...,\frac{1}{r
}^{opt.success}\label{50}\\
&  \leq\frac{1}{r}+\frac{1}{2r}\min_{j=1,...,r}\left\{  \sum_{i=1,...,r
\left\Vert \rho_{i}-\rho_{j}\right\Vert _{1}\right\} \nonumber\\
&  \leq\frac{1}{r}\left(  1+\frac{1}{r}\sum_{1\leq i<j\leq r}\left\Vert
\rho_{i}-\rho_{j}\right\Vert _{1}\right)  .\nonumber
\end{align}

\end{corollary}

\subsection{Example}

For the numerical comparison of the new lower bound (\ref{15})--(\ref{17}) and
the new upper bound (\ref{32}) with the known lower bounds (\ref{28.1
)--(\ref{28.3}) and the known upper bounds (\ref{30}), (\ref{40})--(\ref{42}),
we analyze the discrimination between the following three equiprobable qubit
states
\begin{align}
\rho_{1}  &  =\frac{7}{8}\left\vert 0\right\rangle \left\langle 0\right\vert
+\frac{1}{8}\left\vert 1\right\rangle \left\langle 1\right\vert ,\text{
\ \ }\rho_{2}=\frac{5}{8}\left\vert 0\right\rangle \left\langle 0\right\vert
+\frac{3}{8}\left\vert 1\right\rangle \left\langle 1\right\vert ,\label{51}\\
\rho_{3}  &  =\frac{3}{4}\left\vert 0\right\rangle \left\langle 0\right\vert
+\frac{1}{4}\left\vert 1\right\rangle \left\langle 1\right\vert .\nonumber
\end{align}
In this case, $\left\Vert \rho_{1}-\rho_{2}\right\Vert _{1}=\frac{1}{2},$
$\left\Vert \rho_{1}-\rho_{3}\right\Vert _{1}=\frac{1}{4},$ $\left\Vert
\rho_{2}-\rho_{3}\right\Vert _{1}=\frac{1}{4}$ an
\begin{align}
\sum_{1\leq i<j\leq3}\left\Vert \rho_{i}-\rho_{j}\right\Vert _{1}  &
=1,\text{ \ \ }\min_{j=1,2,3}\left\{  \sum_{i=1,2,3}\left\Vert \rho_{i
-\rho_{j}\right\Vert _{1}\right\}  =\frac{1}{2},\label{52}\\
\max_{j=1,2,3}\left\{  \sum_{i=1,2,3}\left\Vert \rho_{i}-\rho_{j}\right\Vert
_{1}\right\}   &  =\frac{3}{4}.\nonumber
\end{align}
Therefore, in the case considered, the values of the new bounds (\ref{32}),
(\ref{15}), (\ref{17}) are equal t
\begin{align}
\mathfrak{L}_{1,new}  &  =\frac{19}{48}=0.3958,\text{ \ \ }\mathfrak{L
_{2,new}=\frac{7}{18}\simeq0.3889,\label{53}\\
Q_{new}  &  =\frac{4}{9}\simeq0.4444,\nonumber
\end{align}
while the values of the known bounds (\ref{30}), (\ref{40}), (\ref{28.1}) are
\begin{equation}
Q_{4}=\frac{5}{12}\simeq0.4166,\text{ \ \ }Q_{2}=\frac{7}{12}\simeq
0.5833,\text{ \ \ }\mathfrak{L}_{1}=\frac{1}{3}. \label{54
\end{equation}

Note that since states (\ref{51}) mutually commute and are of the form
$\rho_{i}=\sum\limits_{n=0,1}\lambda_{n}^{(i)}\left\vert n\right\rangle
\left\langle n\right\vert ,$ $i=1,2,3,$ the optimal success probability
\cite{4
\begin{equation}
\mathrm{P}_{\rho_{1},\rho_{2},\rho_{_{3}}|\frac{1}{3},\frac{1}{3},\frac{1}{3
}^{opt.success}=\frac{1}{3}\sum\limits_{n=0,1}\left(  \max_{i=1,2,3
\lambda_{n}^{(i)}\right)  =\frac{5}{12}=Q_{4}\simeq0.4166. \label{55
\end{equation}

For the calculation of the lower bounds (\ref{28.2}), (\ref{28.3}) and the
upper bounds (\ref{41}), (\ref{42}) for equiprobable states (\ref{51}), we
calculate fidelities $F_{ij}:=\left\Vert \sqrt{\rho_{i}}\sqrt{\rho_{j
}\right\Vert _{1}$ for states (\ref{51})
\begin{align}
F_{12}  &  =\frac{\sqrt{35}+\sqrt{3}}{8}\simeq0.9560,\text{ \ \ }F_{13
=\frac{\sqrt{42}+\sqrt{2}}{8}\simeq0,9868,\label{56}\\
F_{23}  &  =\frac{\sqrt{30}+\sqrt{6}}{8}\simeq0,9909,\nonumber
\end{align}
and the trac
\begin{equation}
\mathrm{tr}\left[  \sqrt
{\textstyle\sum\limits_{i=1,2,3}}
\rho_{i}^{2}}\right]  =\frac{\sqrt{110}+\sqrt{14}}{8}\simeq1,7787. \label{57
\end{equation}
Therefore,
\begin{align}
Q_{3}  &  =1-\frac{1}{9}\sum_{1\leq i<j\leq3}F_{ij}^{2}\simeq0.6812,\text{
\ \ \ }Q_{5}=\frac{1}{3}\mathrm{tr}\left[  \sqrt
{\textstyle\sum\limits_{i=1,2,3}}
\rho_{i}^{2}}\right]  \simeq0,5929,\label{58}\\
\mathfrak{L}_{2}  &  =1-\frac{1}{3}\sum_{1\leq i<j\leq3}F_{ij}\simeq
0,0221,\text{ \ \ \ }\mathfrak{L}_{3}=\text{\ }\left(  Q_{5}\right)
^{2}\simeq0.3515.\nonumber
\end{align}
From (\ref{53})--(\ref{58}) it follows that, in the case considered
\begin{equation}
\mathrm{P}_{\rho_{1},\rho_{2},\rho_{_{3}}|\frac{1}{3},\frac{1}{3},\frac{1}{3
}^{opt.success}=Q_{4}<Q_{new}<Q_{2}<Q_{5}<Q_{3} \label{59
\end{equation}
and
\begin{equation}
\mathrm{P}_{\rho_{1},\rho_{2},\rho_{_{3}}|\frac{1}{3},\frac{1}{3},\frac{1}{3
}^{opt.success}>\mathfrak{L}_{1,new}>\mathfrak{L}_{2,new}>\mathfrak{L
_{3}>\mathfrak{L}_{1}>\mathfrak{L}_{2}, \label{60
\end{equation}
that is, the values of the new lower bounds $\mathfrak{L}_{new,1},$
$\mathfrak{L}_{new,2}$ and the new upper bound $Q_{new}$ are tighter than the
values of the known lower bounds (\ref{28.1})--(\ref{28.3}) and the known
upper bounds (\ref{40})--(\ref{42}), respectively.

\section{Acknowledgments}

The author is grateful to A. S. Holevo and Min Namkung for useful discussions.

\section{Conclusions}

In the present article, for the optimal success probability (\ref{4}), we find
for all $r\geq2:$ (i) the new general lower bounds (Theorem 2) and specify
their relation (Proposition 1) to the general lower bounds known in the
literature; (ii) the new general upper bound (Theorem 3) and specify its
relation (Proposition 2) to the known general upper bounds.

We also present the example where the values of the new general analytical
bounds, lower and upper, on the optimal success probability are tighter than
the values of the known lower bounds (\ref{28.1})--(\ref{28.3}) and the known
upper bounds (\ref{40})--(\ref{42}), respectively.

The new upper bound (\ref{32}) on the optimal success probability has the form
explicitly generalizing to $r>2$ the Helstrom bound in (\ref{5}) and is easily
calculated. For $r=2$, each of our new bounds, lower and upper, reduces to the
Helstrom\ bound in (\ref{5}), and this proves in the other way the Helstrom
result (\ref{6}).


\section{Conclusion}
In this paper, we propose the Robust Pose Benchmark (COCO-C, MPII-C, OCHuman-C) and rigorously evaluate the performance of current state-of-the-art models on corrupted images. Based on the benchmark, we have some interesting conclusions and examine some useful techniques to improve model robustness. We envision this work will draw the community's attention to this challenging problem and promote the development of robust pose estimators. To improve the model robustness, we propose AdvMix, a novel model-agnostic data augmentation method, to learn to mix up randomly augmented images. Our method significantly improves the robustness of most existing pose estimation models across a wide range of common corruptions while maintaining performance on clean data without extra inference computational overhead.

\noindent \textbf{Acknowledgement.} Ping Luo was supported by the Research Donation from SenseTime and the General Research Fund of HK No.27208720.

{\small
\bibliographystyle{ieee_fullname}

\section{Evaluation on Robust Pose Benchmark}
\subsection{Experimental Setup}
We extensively evaluate the performance of the state-of-the-art methods on the proposed Robust Human Pose Benchmark. To assess the robustness of human pose estimators, the models are only trained on clean data (\textit{e}.\textit{g}.  COCO) and then evaluated on corrupted data (\textit{e}.\textit{g}.  COCO-C). Note that OCHuman dataset is only designed for validation, we follow the common settings~\cite{ochuman} to train the models on COCO and evaluate on OCHuman-C. For top-down methods (\textit{i}.\textit{e}. \textbf{SBL}~\cite{simple}, \textbf{HRNet}~\cite{hrnet}) with input size $256 \times 192$ and $384 \times 288$, we use the officially trained checkpoints~\footnote{https://github.com/leoxiaobin/deep-high-resolution-net.pytorch}, and then directly evaluate their robustness performance on on COCO-C and MPII-C datasets. Since the pre-trained models with image size $128 \times 96$ are not publicly available, we retrain these models following the official training settings. For bottom-up methods (\textit{i}.\textit{e}. \textbf{PifPaf}~\cite{pifpaf}, \textbf{HigherHRNet}~\cite{higherhrnet}), we also follow the official codes and settings. The pre-trained models of PifPaf~\footnote{https://github.com/vita-epfl/openpifpaf/tree/v0.10.0} and HigherHRNet~\footnote{https://github.com/HRNet/HigherHRNet-Human-Pose-Estimation} are directly used for evaluating the robustness using on COCO-C and OCHuman-C. The results are reported in Table \ref{tab:benchmark_all}.

\subsection{Benchmarking Conclusions}

\subsubsection{Pose estimation methods performance}
\textbf{Top-down and bottom-up models show similar performance degradation tendency on different corruptions.} As shown in Figure \ref{fig:method_robustness}, the model robustness to different types of corruption varies a lot. However, performance degradation across different models are similar. For example, all models are more robust to weather or brightness changes, while less robust to motion or zoom blur.

\begin{figure*}[t]
\centering
\includegraphics[width=0.95\textwidth]{figs/Performance_Improvement.pdf}
\caption{(a) Performance improvements of AdvMix on COCO-C across different corruption types. (b) Performance improvements of AdvMix on COCO-C across different corruption severities. The results are obtained with input sizes as $256 \times 192$. Input sizes for top-down (TD) methods are $256 \times 192$, while for bottom-up (BU) methods (\textit{e}.\textit{g}.  HigherHRNet) is $512 \times 512$.}
\label{fig:method_robustness}
\end{figure*}

\begin{figure*}[t]
\centering
\includegraphics[width=0.90\textwidth]{figs/topdown_resolution_network_merge.pdf}
\caption{Performance of top-down and bottom-up models with different backbones and input sizes on COCO-C and OCHuman-C. The line graph shows performance on clean data (AP$^*$), and the bar graph denotes robustness score (rPC). }
\label{fig:arch_robustness}
\end{figure*}

\subsubsection{Backbone network performance}
\textbf{Robustness increases with model capacity.} With the same pose estimation methods, the robustness of different backbone networks varies a lot and increases with model capacity.
In Figure~\ref{fig:arch_robustness}, the line graph shows the performance on clean data (AP$^*$), while the bar graph shows the robustness score (rPC). The color denotes different input image resolutions. We observe that with the same input resolution, a model with higher capacity (HRNet) is generally more accurate and more robust than that with lower capacity (ResNet). This indicates that robustness can be improved by using a stronger backbone network.

\section{Robustness Enhancement with AdvMix}

\begin{table}[tbh]
    \centering
    \caption{\small \textbf{Comparisons} between standard training and AdvMix on COCO-C. For top-down approaches, results are obtained with detected bounding boxes of \cite{hrnet}. We see that mPC and rPC are greatly improved, whilst clean performance AP$^*$ can be preserved.}
    \scalebox{0.8}{
    \begin{tabular}{c|c|c|c|c|c}
\hline
Method & Backbone &  Input size &AP$^*$ & mPC & rPC\\
\hline
Standard &ResNet-50 & $256 \times 192$
&\textbf{70.4} & 47.8 &67.9 \\
\textbf{AdvMix}&ResNet-50 & $256 \times 192$
&70.1 & \textbf{50.1} & \textbf{71.5}\\  \hline
Standard &ResNet-101 & $256 \times 192$
&\textbf{71.4} & 49.6 & 69.5 \\
\textbf{AdvMix}&ResNet-101 & $256 \times 192$
&71.3 & \textbf{52.3} & \textbf{73.3}\\ \hline
Standard &ResNet-152 & $256 \times 192$
& 72.0 & 50.9 &  70.7 \\
\textbf{AdvMix}&ResNet-152 & $256 \times 192$
& \textbf{72.3} & \textbf{53.2} & \textbf{73.6}\\
\hline
Standard&HRNet-W32 & $256 \times 192$
&74.4 & 53.0 & 71.3\\
\textbf{AdvMix}&HRNet-W32& $256 \times 192$
&\textbf{74.7}&\textbf{55.5}&\textbf{74.3} \\ \hline
Standard&HRNet-W48 & $256 \times 192$
& 75.1 & 53.7 & 71.6 \\
\textbf{AdvMix}&HRNet-W48& $256 \times 192$
&\textbf{75.4}&\textbf{57.1}&\textbf{75.7} \\ \hline \hline
Standard&HrHRNet-W32 & $512 \times 512$
&67.1 &39.9	&59.4\\
\textbf{AdvMix}&HrHRNet-W32& $512 \times 512$
&\textbf{68.3} &\textbf{45.4}	&\textbf{66.5} \\
\hline
    \end{tabular}}
    \label{tab:advmix_coco}
\end{table}

\begin{table}[tbh]
    \centering
    \caption{\small \textbf{Comparisons} between standard training and AdvMix of representative top-down methods on MPII-C. We draw similar conclusion that AdvMix is effective to increase mPC and rPC, whilst maintaining clean performance PCKh$^*$.}
    \scalebox{0.8}{
    \begin{tabular}{c|c|c|c|c|c}
\hline
Method & Backbone & Input size & PCKh$^*$ & mPC & rPC\\
\hline
Standard&ResNet-50 &$256 \times 256$
&88.5 &77.5	&87.6 \\
\textbf{AdvMix}&ResNet-50&$256 \times 256$
& \textbf{88.9} & \textbf{82.0} & \textbf{92.3}\\\hline
Standard&ResNet-101 &$256 \times 256$
&89.1 &78.6	& 88.3 \\
\textbf{AdvMix}&ResNet-101&$256 \times 256$
& \textbf{89.4} & \textbf{82.8} & \textbf{92.5}\\
\hline
Standard&HRNet-W32& $256 \times 256$
& 90.3 &80.1&88.7\\
\textbf{AdvMix}&HRNet-W32 &$256 \times 256$
&\textbf{90.5} &\textbf{83.9} & \textbf{92.7} \\ \hline
    \end{tabular}}
    \label{tab:advmix_mpii}
\end{table}

\subsection{Implementation Details}
For adversarial training, we simply reuse the existing publicly available well-trained checkpoints and retrain the models with AdvMix on clean data. We simply follow the same training settings (\textit{i}.\textit{e}.  the same hyper-parameters, learning rate, and total training epochs) as the official codes. By default, the initial learning rates for the augmentation generator and human pose estimator is 0.001. We decay the learning rate by the factor of 10 at the 170-th epoch, and 200-th epoch. The adversarial training ends at 210-th epoch. We adopt ADAM optimizer to train both the augmentation generator and the human pose estimator.
All experiments are conducted using PyTorch~\cite{paszke2017automatic} on NVIDIA TITAN X Pascal GPUs.

\begin{table*}[h]
    \centering
    \caption{\small Results of directly augmenting with the test-time corruption types (Transfer). We use $^\dagger$ to denote the selected corruption types (\textit{e}.\textit{g}.  Gaussian noise, Defocus blur, Snow, and Contrast) are used as data augmentation during training. }

    \scalebox{0.69}{
    \begin{tabular}{c|c|c|ccc|cccc|cccc|cccc}
\hline
Method & AP$^*$ & mPC & Gauss& Shot & Impulse & Defoucs & Glass & Motion & Zoom & Snow & Frost & Fog & Bright & Contrast & Elastic & Pixel & JPEG \\
\hline
Standard&76.6&53.3&51.1&54.1&52.5&46.7&46.2&43.4&18.8&56.2&59.7&66.2&71.8&54.5&61.1&59.8&57.1 \\
Transfer&\textcolor{gray}{75.4}&56.2&58.7$^\dagger$&60.3&59.7&55.0$^\dagger$&48.3&44.3&19.0&65.6$^\dagger$&61.7&\textcolor{gray}{65.8}&\textcolor{gray}{70.9}&58.9$^\dagger$&\textcolor{gray}{60.3}&61.8&\textcolor{gray}{52.2} \\
\hline
AdvMix&\textbf{77.1}&55.9&54.7&57.9&56.7&50.4&49.1&44.2&\textcolor{gray}{18.2}&59.8&62.4&68.2&73.2&56.0&62.6&63.4&61.5 \\
AdvMix+Stylized&76.5&\textbf{56.5}&54.5&57.5&56.4&50.4&49.3&46.1&20.0&62.3&63.0&67.6&72.9&59.0&62.5&63.9&61.8 \\
\hline
    \end{tabular}}
    \label{tab:type}
\end{table*}

\begin{table}[h]
    \centering
    \caption{\small \textbf{Comparisons} with other techniques on COCO-C with HRNet-W32 backbone. The results are obtained using ground truth bounding boxes. We can observe that AdvMix significantly outperforms the state-of-the-art methods on robustness, while preserving the performance of clean AP.}

    \scalebox{0.8}{
    \begin{tabular}{c|c|c|c}
\hline
Method & AP$^*$ & mPC & rPC\\
\hline
Standard&
76.6&53.3&69.6\\
FPD~\cite{zhang2019fast}&
77.3&54.0&69.9\\
FPD$^\dagger$~\cite{zhang2019fast}&
76.6&53.4&69.7\\
Grid-Mask~\cite{gridmask}&
76.4&54.8&71.7\\
AutoAugment~\cite{autoaugment}&
76.2&54.2&71.1\\
Stylized Only~\cite{imagenet_benchmark,det_benchmark}&
67.5& 46.7 & 69.1\\
Stylized~\cite{imagenet_benchmark,det_benchmark}&
76.1&54.8&72.0\\ \hline
\textbf{AdvMix} &
\textbf{77.1}& 55.9& 72.5\\
Stylized + \textbf{AdvMix}&
76.5& \textbf{56.5}&\textbf{73.8}\\
\hline
    \end{tabular}}
    \label{tab:method_comparison}
\end{table}

\begin{table}[h]
    \centering
    \caption{\small \textbf{Ablation} study on COCO-C with HRNet-W32 backbone. The results are obtained using ground truth bounding boxes. We see that AdvMix performs better than other augmentation composing approaches on robustness by large margin.}

    \scalebox{0.8}{
    \begin{tabular}{c|c|c|c}
\hline
Method & AP$^*$ & mPC & rPC\\
\hline
SequentialMix &
76.2&54.6&71.6\\
EqualMix& 76.6 & 54.2 & 70.7 \\
DirichletMix~\cite{augmix}& 76.5 & 54.4 & 71.1 \\ \hline
\textbf{AdvMix}-Image & \textbf{77.5}& 55.1& 71.2\\
\textbf{AdvMix} &
77.1& \textbf{55.9}& \textbf{72.5} \\
\hline
    \end{tabular}}
    \label{tab:ablation}
\end{table}

\begin{table}[h]
    \centering
    \caption{Effect of knowledge distillation.}

    \scalebox{0.8}{
    \begin{tabular}{c|c|c|c|c}
\hline
Method & dataset & AP$^*$/ PCKh$^*$ & mPC & rPC\\
\hline
\textbf{AdvMix} w/o KD  & COCO-C & 76.8 & 55.9 & 72.7 \\
\textbf{AdvMix} & COCO-C &
\textbf{77.1}& 55.9 & 72.5 \\
\hline
\textbf{AdvMix} w/o KD  & MPII-C & 89.9 & 81.7 & 90.9 \\
\textbf{AdvMix} & MPII-C &
\textbf{90.5} &\textbf{83.9} & \textbf{92.7} \\
\hline
    \end{tabular}}
    \label{tab:ablation_kd}
\end{table}

\subsection{Quantitative Results}
As shown in Table~\ref{tab:advmix_coco} and Table~\ref{tab:advmix_mpii}, we find that the proposed method significantly improves the model robustness while the performance on clean data is maintained or slightly improved. We also observe that AdvMix significantly boosts the robustness performance for bottom-up methods ($39.9$ to $45.4$ mPC) and the robustness gain is larger than that of top-down methods shown in Table~\ref{tab:advmix_coco}.
Figure~\ref{fig:method_robustness}(a) shows the performance degradation and the robustness improvement of different models across different corruptions. Performance degradation among different models shows a similar tendency. Models are generally more robust to brightness and weather changes, while less robust to motion and zoom blur. AdvMix performs better than baseline by different corruption types and the gain of noise and digital distortion are larger than other corruption types.
Comparing the performance across different corruption severity in Figure~\ref{fig:method_robustness}(b), we observe that AdvMix consistently improves over the baseline, and the gain gets larger for severer corruption.

We compare AdvMix with other state-of-the-art methods. FPD~\cite{zhang2019fast} proposes to improve the performance of lightweight models through knowledge distillation, and the teacher net is more sophisticated than the student net. FPD$^\dagger$ is our implementation with the same setting as FPD except that the teacher net uses the same architecture as the student net. Stylized dataset~\cite{style_transfer} is also used as a technique to improve robustness. As shown in Table~\ref{tab:method_comparison}, we can see that though FPD can boost the performance on clean data, the improvement of robustness is limited.
We also find that only using knowledge distillation (teacher use the same architecture in FPD$^\dagger$) without AdvMix will not improve the performance on clean data and model robustness.
AdvMix can not only improve the mPC, but also slightly improve the performance on clean data, outperforming operating AutoAugment or Grid-Mask separately.
Finally, we also prove that AdvMix can be combined with the data stylizing technique and further enhance the robustness on corrupted images while almost maintaining the performance of clean data.

\subsection{Ablation Studies}
\label{sec:exp_ablation}

We conduct ablation studies on COCO-C dataset using HRNet-W32 backbone with input size $256 \times 192$ to validate the effectiveness of the augmentation composition method in AdvMix.
SequentialMix simply composes augmentation operations in a chain and applies them sequentially. EqualMix mixes the augmented images from various augmentation chains using equal weights. DirichletMix follows \cite{augmix} to sample the mix weights from Dirichlet distribution to mix the augmented images. In comparison, AdvMix uses the augmentation generator to \textit{learn} to generate the per-pixel mix weights adversarially. Different from AdvMix, the augmentation generator of AdvMix-Image outputs per-image mix weights rather than per-pixel mix weights.
We observe that AdvMix significantly outperforms EqualMix and DirichletMix on model robustness (mPC and rPC) by a large margin, demonstrating the effectiveness of the proposed adversarial weights learning. Meanwhile, AdvMix with learnt per-pixel composition is more robust than learnt per-image composition (AdvMix-Image) since per-pixel composition could generate more diverse and fine-grained training samples to boost robustness performance, \textit{i}.\textit{e}. , per-image mixing can hardly composite image in region-level, \textit{e}.\textit{g}. , person and background, while per-pixel mixing with learnt weights can pay more attention to important regions.
As illustrated in Table~\ref{tab:ablation_kd}, knowledge distillation prevents over-fitting to the induced noise patterns from creeping into the feature space and helps maintain or improve the performance on clean data.

Since we focus on how to improve robustness on \textbf{unseen} data, the corruption images in the benchmark should \emph{not} be encountered while training. However, to investigate the phenomenon if we augment the images with the test-time corruption type in the benchmark, we select one corruption type (\textit{i}.\textit{e}.  Gaussian noise, Defocus blur, Snow and Contrast) from each corruption category (\textit{i}.\textit{e}.  noise, blur, weather, and digital) as augmentation types. As illustrated in Table~\ref{tab:type}, we can observe that augmenting the training samples with test-time corruptions can boost the robustness, but the performance on clean data decreases a lot. Meanwhile, augmenting with some specific types of noises improves the performance on the target noises, but it does not always generalize to other unseen corruption types, even within the same corruption category (\textit{e}.\textit{g}.  augmentation of Snow does not contribute to improving the robustness of Fog and Brightness). By contrast, AdvMix training with stylized data achieves the best mPC. It consistently improves the performance across all corruption types, while maintaining similar clean data performance as standard training.

\subsection{Qualitative Comparison}
In Figure \ref{fig:qualitative_results}, we visualize the results of images with different types of image corruptions, \textit{i}.\textit{e}.  impulse noise, Motion blur, Brightness, and JPEG compression.
We observe that 1) the standard pose models suffer a large performance drop on corrupted data, and 2) models trained with AdvMix perform consistently better than the baseline methods on various corruptions.

\begin{figure}[h]
\centering
\includegraphics[width=0.45\textwidth]{figs/visualization.pdf}
\caption{Qualitative comparisons between HRNet without and with AdvMix. For each image triplet, the images from left to right are ground truth, predicted results of Standard HRNet-W32, and predicted results of HRNet-W32 with AdvMix.}
\label{fig:qualitative_results}
\end{figure}

\section{Introduction}

Human pose estimation (HPE) is a fundamental task for action recognition and video surveillance \cite{action_1, action_2, pose_surveilance}. Although convolutional neural networks (CNNs)  achieved great progress \cite{CPM, simple, hrnet, openpose, asso_embedding, higherhrnet} on challenging datasets \cite{coco, mpii, ochuman}, which only contain clean and high-resolution images, deploying models in the real world requires not only good performance on clean data, but also robustness to commonly occurring image corruptions. Unfortunately, unlike the human vision system, current pose estimators suffer severe performance drop when images are corrupted. Therefore, analyzing and enhancing the robustness of pose estimators are important and are the purposes of this work.

\begin{figure}[t]
\centering
\includegraphics[width=0.45\textwidth]{figs/1st_page_v4.pdf}
\caption{\small \textbf{Comparisons} between HRNet without and with AdvMix, which significantly improves the standard HRNet in images with subtle JPEG compression (1st row) and Frost artifacts (2nd row). Please compare results of ground truth (left), HRNet (middle), and our HRNet+AdvMix (right).}
\label{fig:vis}
\end{figure}

Recent work on the robustness of CNNs can be categorized into adversarial robustness and corruption robustness. Firstly, adversarial robustness such as \cite{adv_robust_1, adv_robust_2, adv_robust_3, adv_robust_4, adv_robust_5, adv_benchmark_pose} mainly focuses on the robustness of neural networks towards worst-case adversarial examples, which are carefully designed by attackers. However, the samples generated by maximizing recognition errors (worst cases) may not naturally exist in real applications.
Secondly, in contrast, corruption robustness pays attention to practical real-world image distortions such as noise, blur, and bad weather conditions. This work focuses on corruption robustness and tries to figure out how image quality affects the performance of pose estimators.

Although there are recent studies on common robustness for classification, object detection, and semantic segmentation \cite{imagenet_benchmark, det_benchmark, seg_benchmark}, we are the first to conduct comprehensive evaluations for human pose estimation. Unlike classification, detection, and segmentation, HPE uses a blend of classification and regression methods to model the structures of the human body, making it a challenging and collaborative field that is worthy of special investigations.

Inspired by~\cite{imagenet_benchmark}, we evaluate advanced human pose estimators on a full spectrum of \textbf{unseen} corruption types, which are not encountered in model training (\textit{i}.\textit{e}.  CNNs are trained on \textit{clean} images, while evaluated on corrupted images).
Our main contribution is to establish the robust pose benchmarks, consisting of three challenging datasets including COCO-C, MPII-C, and OCHuman-C.
Extensive evaluations on these benchmarks show the \textbf{weaknesses} of both existing top-down and bottom-up pose estimators. (1) The state-of-the-art pose estimators suffer severe performance drop on corrupted images as shown in Fig.\ref{fig:vis} (\textit{e}.\textit{g}.  accuracy of HRNet-W32 with image size $256 \times 192 $ is reduced from 74.4 to 53.0 on COCO-C). (2) By comparing various types of image corruptions, we find that models are generally more robust to brightness and weather changes, while less robust to motion and zoom blur. (3) The model robustness would increase by increasing model capacity.

This work aims at improving robustness against different \textbf{unforeseen} corruptions. Extensive empirical evaluations on the proposed benchmarks help us screen a collection of useful data augmentation techniques to improve model robustness under severe corruptions. How can we make full use of these techniques and combine random augmented images to further improve the generalization of human pose estimators towards corruptions in our benchmark?
To address this problem, we propose a novel data augmentation algorithm AdvMix, which \textit{learns} to automatically combine augmented images. Specifically, AdvMix jointly trains two neural networks in an adversarial manner, \textit{i}.\textit{e}.  an augmentation generator and a human pose estimator. The generator produces weights to mix up randomly augmented strategies, while the pose estimator attempts to learn robust visual representation from harder training samples.
\jh{To reduce the effect of AdvMix-induced noise patterns}, we propose to use a pre-trained teacher pose estimator to transfer knowledge learned from \textbf{clean} training data in advance towards the target human pose estimator.
Extensive evaluations demonstrate that AdvMix significantly improves model robustness on diverse image corruptions while maintaining performance on clean data.

Our main \textbf{contributions} can be summarized as follows.
\begin{itemize}
\item We propose three robust benchmarks COCO-C, MPII-C, and OCHuman-C, and demonstrate that both top-down and bottom-up pose estimators suffer severe performance drop on corrupted images, drawing the community's attention to this problem. This is also a critical problem to be addressed for deploying models in real applications.

\item With extensive experiments, we have many interesting conclusions that would help improve the accuracy and robustness of future works.

\item We propose a novel adversarial data augmentation method together with knowledge distillation, termed AdvMix, which is model-agnostic and easy-to-implement. It significantly improves the robustness of pose estimation models while maintaining or slightly improving the performance on the clean data, without extra inference computational overhead.

\end{itemize}
\section{Introduction}

Human pose estimation (HPE) is a fundamental task for action recognition and video surveillance \cite{action_1, action_2, pose_surveilance}. Although convolutional neural networks (CNNs)  achieved great progress \cite{CPM, simple, hrnet, openpose, asso_embedding, higherhrnet} on challenging datasets \cite{coco, mpii, ochuman}, which only contain clean and high-resolution images, deploying models in the real world requires not only good performance on clean data, but also robustness to commonly occurring image corruptions. For example, while tracking and estimating the keypoints of a moving person in outdoor environments, current pose estimators suffer severe performance drop due to the noise or blur caused by weather conditions or camera systems. Therefore, analyzing and enhancing the robustness of pose estimators are important and are the purposes of this work.

\begin{figure}[t]
\centering
\includegraphics[width=0.45\textwidth]{figs/Overall.pdf}
\caption{\small \textbf{Improvements} of model robustness (mPC) when AdvMix is applied to the state-of-the-art methods.}
\label{fig:vis}
\end{figure}

Unlike previous studies on common robustness for classification, detection and segmentation \cite{imagenet_benchmark, det_benchmark, seg_benchmark}, human pose estimation uses a blend of classification and regression methods to model the structures of the human body, making it a challenging and collaborative field that is worthy of special investigations. The key challenges of robust human pose estimation are three folds. First, the lack of a benchmark for evaluating the robustness of state-of-the-art human pose estimation methods makes it difficult to construct rigorous comparisons between different models, not to mention to improve model robustness. Second, accuracy of clean data and corrupted data are trade-offs. Improving the robustness of the model while maintaining its performance on clean data is a non-trivial problem. Third, based on the proposed benchmark, we have examined the effectiveness of some data augmentation methods. However, we find that simply applying them sequentially does not achieve desirable performance. How can we effectively combine existing data augmentation techniques to improve the generalization of human pose estimators towards unforeseen corruptions?

Inspired by~\cite{imagenet_benchmark}, we establish the robust pose benchmarks, consisting of three challenging datasets including COCO-C, MPII-C, and OCHuman-C. The benchmark datasets are constructed based on a full spectrum of \textbf{unforeseen} corruption types that are not encountered in model training (\textit{i}.\textit{e}.  CNNs are trained on \textit{clean} images, while evaluated on corrupted images).
Extensive evaluations on these benchmarks show the weakness of both existing top-down and bottom-up pose estimators. (1) The state-of-the-art pose estimators suffer severe performance drop on corrupted images. (2) Models are generally more robust to brightness and weather changes, while less robust to motion and zoom blur. (3) The model robustness would increase by increasing model capacity.

Empirical evaluations on the proposed benchmarks help us screen a collection of useful data augmentation techniques to improve model robustness under severe corruptions. In order to make full use of these techniques and achieve optimal performance on \textbf{unforeseen} noisy data, we propose an augmentation generator, which \textit{learns} to automatically combine augmented images. Specifically, we jointly train two neural networks in an adversarial manner, \textit{i}.\textit{e}.  an augmentation generator and a human pose estimator. The generator produces weights to mix up randomly augmented images, while the pose estimator attempts to learn robust visual representation from harder training samples.

It is worth noting that
the compositions produced by the augmentation generator may drift far from original images and such induced noise patterns may be harmful to performance on clean data. To reduce this negative impact, we propose to use a pre-trained teacher pose estimator to transfer structure knowledge learned from entire \textbf{clean} training data towards the target human pose estimator. Different from previous knowledge distillation methods that use a stronger network as the teacher model, our teacher pose estimator shares the same architecture as the target pose estimator.
Extensive evaluations demonstrate that AdvMix significantly improves model robustness on diverse image corruptions while maintaining performance on clean data. The augmentation generator and the teacher pose estimator are only used for training and will be discarded at the inference stage, and thus introducing no computational overhead at inference time. Meanwhile, as shown in Fig.~\ref{fig:vis}, our method is model-agnostic and is proved to be effective for various state-of-the-art pose estimation models.

Our main \textbf{contributions} can be summarized as follows.
\begin{itemize}
\item We propose three robust benchmarks COCO-C, MPII-C, and OCHuman-C, and demonstrate that both top-down and bottom-up pose estimators suffer severe performance drop on corrupted images, drawing the community's attention to this problem.

\item With extensive experiments, we have many interesting conclusions that would help improve the accuracy and robustness of future works.

\item We propose a novel adversarial data augmentation method together with knowledge distillation, termed AdvMix, which is model-agnostic and easy-to-implement. It significantly improves the robustness of pose estimation models while maintaining or slightly improving the performance on the clean data, without extra inference computational overhead.

\end{itemize}
\section{Methods}
\subsection{Robust Pose Benchmark}

\begin{figure*}[t]
\centering
\includegraphics[width=1.0\textwidth]{figs/framework.pdf}
\caption{\textbf{Overview of AdvMix.} Our framework consists of two modules: the adversarial augmentation module and the knowledge distillation module. The adversarial augmentation module contains an augmentation generator and a pose estimator. For example, given two (\textit{i}.\textit{e}.  $K=2$) differently augmented images, the augmentation generator estimates the corresponding attention maps $\mathcal{M}$, and mixes them up to the final image $I_{mix}$, while the pose estimator generates keypoint heatmaps. They are trained in an adversarial manner following Eq.(\ref{optimizaton_process}). The robustness of the pose estimator can be significantly improved if the generator cannot confuse it. To reduce the effect of induced noise patterns, we use a teacher pose estimator for transferring pose structure knowledge in adversarial training. The teacher pose estimator shares the same architecture as the target pose estimator and is pre-trained on the entire clean data.}
\label{fig:framework}
\end{figure*}

\subsubsection{Benchmark Datasets}
The robust pose estimation benchmark is composed of three benchmark datasets: COCO-C, MPII-C, OCHuman-C, which are constructed by applying 15 different types of image corruptions~\cite{imagenet_benchmark} with 5 severity levels to the official \textit{validation} set of COCO~\cite{coco}, MPII~\cite{mpii} and OCHuman~\cite{ochuman}. Therefore, the total number of each benchmarking dataset is $15 \times 5$ times that of the corresponding validation dataset. The types of image corruption are sorted into four main categories (noise, blur, weather, and digital), which are diverse and enormous enough to cover real-world corruptions.

\textbf{COCO-C Dataset.} COCO-C dataset is constructed from COCO~\cite{coco} val2017 set. The COCO dataset contains over 200,000 images and 250,000 person instances, where there are 5000 images for the val2017 set.

\textbf{MPII-C Dataset.} MPII-C dataset is constructed from MPII~\cite{mpii} test set. The MPII dataset consists of images taken from a wide range of real-world activities with full-body pose annotations. There are around 25K images with 40K subjects, where there are 12K subjects for testing and the remaining subjects for the training set.

\textbf{OCHuman-C Dataset.} OCHuman-C is constructed from OCHuman~\cite{ochuman}. The OCHuman dataset focuses on heavily occluded human with comprehensive annotations including bounding-box, humans pose and instance mask, which contains 13360 elaborately annotated human instances within 5081 images.

\subsubsection{Evaluation Metrics}
For COCO~\cite{coco} and OCHuman~\cite{ochuman}, the standard average precision (AP) is used to evaluate the models. In this paper, we use $\text{AP}^*$ to denote the performance on \textbf{clean} data. To evaluate model robustness, we follow~\cite{det_benchmark} to use mean performance under corruption ($\text{mPC}$):

\begin{equation}
    \text{mPC} = \frac{1}{N_c} \sum\limits_{c=1}^{N_c} \frac{1}{N_s} \sum\limits_{s=1}^{N_s} \text{AP}_{c,s}.
    \label{mPC}
\end{equation}
Here, $\text{AP}_{c,s}$ is the average performance measured on corruption type $c$ under severity level $s$. $N_c=15$ and $N_s=5$ are the numbers of corruption types and severity levels, respectively. To assess the robustness of models, we define the relative performance under corruption ($\text{rPC}$) as follows:

\begin{equation}
    \text{rPC} = \frac{\text{mPC}}{\text{AP}^*}.
    \label{rAP}
\end{equation}

For MPII~\cite{mpii} dataset, the official evaluation metric is PCKh. Similarly, we use $\text{PCKh}^*$ to denote the performance on clean data. Similar to mPC and rPC in Eq.(\ref{mPC},\ref{rAP}), mean performance under corruption ($\text{mPC}$) and relative mean performance under corruption ($\text{rPC}$) are introduced as follows:

\begin{equation}
    \text{mPC} = \frac{1}{N_c} \sum\limits_{c=1}^{N_c} \frac{1}{N_s} \sum\limits_{s=1}^{N_s} \text{PCKh}_{c,s}
    \label{mPC_mpii}
\end{equation}

\begin{equation}
    \text{rPC} = \frac{\text{mPC}}{\text{PCKh}^*}.
    \label{rPCKh}
\end{equation}

\subsection{Adversarial Augmentation Mix (AdvMix)}

\subsubsection{Augmentation Generator}

As shown in Figure \ref{fig:framework}, given an image $I$, we first randomly generate $K$ differently augmented images $\mathcal O_k(I)$ using parallel augmentation strategies $\mathcal O_k$. We use $K=2$ by default in our implementation. Together with the original image $I$, we get a set of $K + 1$ proposal images. The augmentation generator $\mathcal{G}(\cdot, \theta)$ is applied to output the normalized attention maps $\mathcal{M}^{(K + 1) \times H \times W}$, where $H$ and $W$ are the image resolution. The attention maps $\mathcal{M}$ are used as the weights to mix up the proposal images, following Eq.(\ref{eq:mixed_image}), where $\odot$ is the Hadamard product.

\begin{equation}
    I_{mix} =\mathcal{M}_0 \odot {I} + \sum_{k=1}^{K}{\mathcal{M}_k \odot \mathcal O_k(I)}.
    \label{eq:mixed_image}
\end{equation}

Based on the benchmark (Table.\ref{tab:benchmark_all}), we examine and find a collection of useful techniques to improve the model robustness for pose estimation. Grid-Mask \cite{gridmask} applies information removal to achieve state-of-the-art results in various computer vision tasks. AutoAugment~\cite{autoaugment} automatically searches for a mixture of augmentation policies. It improves both the clean performance and the robustness for image classification~\cite{fourier}. To avoid over-fitting to the test set of held-out corruptions, we manually exclude some operations, such as contrast, color, brightness, and sharpness sub-policies (as they appear in the benchmark). Given an image $I$, we choose Grid-Mask and AutoAugment to generate randomly augmented images $\mathcal O$, and then mix these images through mixing weight from augmentation generator. We adopt the U-Net~\cite{Unet} architecture to build the augmentation generator $\mathcal{G}(\cdot, \theta)$, which generates the attention maps for mixing up randomly augmented images. Please refer to the supplementary for more implementation details.

\subsubsection{Heatmap Regression}

The human body pose is encoded with 2D Gaussian confidence heatmaps, where each channel corresponds to one body keypoint. The objective of human pose estimation network $\mathcal{D}(\cdot, \phi)$ is to minimize the MSE loss between the predicted heatmaps and the ground truth heatmaps $\mathcal{H}_{gt}$:

\begin{equation}
    \mathcal{L}_\mathcal{D^*} = \left\| \mathcal{D}(I_{mix}, \phi) - \mathcal{H}_{gt}\right\|_2^2.
    \label{eq:mse_loss}
\end{equation}

To enhance the supervision to the target pose estimator $\mathcal{D}$, we introduce a teacher pose estimator $\mathcal{T}$ for knowledge distillation and providing softened heatmap labels. Note that $\mathcal{T}$ and $\mathcal{D}$ share the same architecture and $\mathcal{T}$ is pre-trained on the clean training data. The parameters of the $\mathcal{T}$ will be fixed while training. We select MSE loss for knowledge distillation loss as:

\begin{equation}
    \mathcal{L}_{\mathcal{D}_{kd}}= \left\| \mathcal{D}(I, \phi) - \mathcal{T}(I)\right\|_2^2.
    \label{eq:kd_loss}
\end{equation}

We formulate the overall loss function of pose estimator $\mathcal{D}$ while training as:

\begin{equation}
    \mathcal{L}_\mathcal{D} = (1 - \alpha) \mathcal{L}_\mathcal{D^*} + \alpha \mathcal{L}_{\mathcal{D}_{kd}}.
    \label{eq:overall_D_loss}
\end{equation}

where $\alpha$ is the loss weight balancing between MSE loss and knowledge distillation loss. We observe that our model is not sensitive to $\alpha$. Thus we choose $\alpha=0.1$ as the default setting in our experiments.

\subsubsection{Adversarial Training}

The augmentation generator $\mathcal{G}(\cdot, \theta)$ and human pose estimator $\mathcal{D}(\cdot, \phi)$ are trained in an adversarial manner. $\mathcal{G}(\cdot, \theta)$ tries to find the most confusing way to mix up randomly augmented images, while $\mathcal{D}(\cdot, \phi)$ learns more robust features from harder training samples. The optimization objective of augmentation generator is defined as:

\begin{equation}
    \mathcal{L}_{\mathcal{G}} = -\mathcal{L}_{\mathcal{D^*}}.
    \label{eq:G_loss}
\end{equation}

Overall, the whole learning process can be defined as two-player zero-sum game with value function $\mathcal{V}(\mathcal{D}, \mathcal{G})$.

\begin{equation}
    \mathcal{V}(\mathcal{D}, \mathcal{G}) = \mathop{\min}_{\phi} \mathop{\max}_{\theta}\mathop{\mathbb{E}}_{\mathcal{I}\sim\Omega} \mathcal{L}(\mathcal{D}(\mathcal{G}(\mathcal{\mathcal{O}(\mathcal{I}), \theta)}, \phi),\mathcal{H}_{gt}).
    \label{optimizaton_process}
\end{equation}

\begin{table*}[tb]
    \centering
    \caption{\small Pose robustness benchmark for both top-down and bottom-up models on COCO-C, MPII-C and OChuman-C. AP$^*$ and PCKh$^*$ represent the performance on clean data. mPC represents mean performance under all corruptions, while rPC measures the relative performance. The remaining columns are the breakdown APs for different corruptions. We see that performances of existing advanced pose estimators significantly drop when corruptions are presented.}

    \scalebox{0.59}{
    \begin{tabular}{c|c|c|c|a|a|ccc|cccc|cccc|cccc}
\hline
\multicolumn{21}{c}{} \\[-5pt]
\multicolumn{21}{c}{\large Results of top-down methods on COCO-C with the same detection bounding boxes as \cite{hrnet}} \\[5pt]
\hline
Method & Backbone & Input size & AP$^*$ & mPC & rPC & Gauss& Shot & Impulse & Defoucs & Glass & Motion & Zoom & Snow & Frost & Fog & Bright & Contrast & Elastic & Pixel & JPEG \\
\hline
\multirow{6}*{SBL~\cite{simple}}
&\multirow{3}*{ResNet-50}
& $128 \times 96$ &
59.0&40.7&69.0&37.8&39.8&36.9&39.5&38.1&35.3&13.3&39.1&43.0&48.1&54.5&38.5&49.2&50.9&47.1\\
&& $256 \times 192$ & 70.4&47.8&67.9&45.8&48.1&45.6&43.4&42.1&38.8&16.3&49.1&52.5&58.9&65.5&47.2&56.7&55.2&51.7\\
&& $384 \times 288$ &72.2&47.7&66.1&45.2&47.8&45.9&42.9&42.1&38.1&16.3&49.9&53.2&60.0&66.8&48.2&57.3&53.2&49.2\\
&\multirow{3}*{ResNet-101}
& $128 \times 96$&
61.1&42.6&69.7&40.4&42.6&39.6&40.7&39.9&36.8&14.5&41.3&45.2&49.6&56.8&38.6&51.1&53.2&49.1\\
&& $256 \times 192$ & 71.4&49.6&69.5&47.8&50.1&47.2&45.1&43.8&40.3&17.6&50.9&54.9&60.9&67.0&49.7&58.1&57.0&53.8 \\
&& $384 \times 288$&
73.6&50.4&68.4&49.2&51.7&49.1&44.8&43.9&40.0&17.7&52.5&56.4&62.7&69.0&51.2&59.0&55.5&52.7\\
\hline
\multirow{6}*{HRNet~\cite{hrnet}}
&\multirow{3}*{HRNet-W32}
& $128 \times 96$&
66.9&47.2&70.6&42.7&45.7&43.0&44.0&43.1&40.9&15.9&47.1&51.4&57.4&63.0&47.5&55.7&57.2&53.4 \\
&& $256 \times 192$ &74.4&53.0&71.3&51.3&54.2&52.6&46.9&46.3&43.5&19.2&55.9&59.1&65.2&70.3&54.1&60.5&59.4&56.9  \\
&& $384 \times 288$ & 75.7&53.7&70.9&51.9&54.7&53.7&47.8&47.1&43.8&19.8&57.9&60.3&66.5&71.6&55.4&61.1&58.1&55.8 \\
&\multirow{3}*{HRNet-W48}
& $128 \times 96$ &
68.6&49.3&71.8&45.9&48.8&46.1&45.3&44.4&42.4&16.5&49.6&54.1&59.8&65.0&49.4&57.3&59.2&55.2
\\
&& $256 \times 192$ &75.1&53.7&71.6&52.5&55.2&53.4&46.8&46.7&43.5&19.1&57.0&60.1&66.4&71.4&55.2&61.1&60.0&57.6\\
&& $384 \times 288$ & 76.3&54.2&71.1&52.8&55.8&54.2&47.6&47.3&43.4&19.5&58.3&60.9&67.5&72.3&56.3&61.6&59.2&57.1 \\
\hline

\multicolumn{21}{c}{} \\[-5pt]
\multicolumn{21}{c}{\large Results of top-down methods on MPII-C} \\[5pt]
\hline
Method& Backbone & Input size & PCKh$^*$ & mPC & rPC & Gauss& Shot & Impulse & Defoucs & Glass & Motion & Zoom & Snow & Frost & Fog & Bright & Contrast & Elastic & Pixel & JPEG \\
\hline
\multirow{3}*{SBL~\cite{simple}}
&Resnet-50 & $256 \times 256$ & 88.5&77.5&87.6&68.9&71.6&68.9&84.2&85.3&83.6&53.7&70.0&75.6&83.1&86.6&69.1&87.8&87.8&86.8 \\
&Resnet-101 & $256 \times 256$ &
89.1&78.6&88.3&70.4&73.1&70.6&84.8&86.0&84.1&53.9&72.2&76.9&84.1&87.0&71.9&88.3&88.4&87.3\\
&Resnet-152&$256 \times 256$ &
89.6&79.6&88.8&74.1&76.3&74.1&85.3&86.5&84.8&54.5&72.1&77.6&84.4&87.7&71.4&88.8&88.8&87.9\\
\hline
HRNet~\cite{hrnet}&HRNet-W32& $256 \times 256$ & 90.3&80.1&88.7&71.5&74.1&72.4&86.0&87.0&85.3&56.0&73.8&78.6&86.1&88.5&74.5&89.5&89.5&88.7 \\
\hline
\multicolumn{21}{c}{} \\[-5pt]
\multicolumn{21}{c}{\large Results of bottom-up methods on COCO-C} \\[5pt]
\hline
Method & Backbone & Input size & AP$^*$ & mPC & rPC & Gauss& Shot & Impulse & Defoucs & Glass & Motion & Zoom & Snow & Frost & Fog & Bright & Contrast & Elastic & Pixel & JPEG \\
\hline
\multirow{3}*{PifPaf~\cite{pifpaf}}
& ShuffleNet V2 & $641 \times 641$ &60.7&32.9&54.2&25.7&27.3&24.6&29.1&28.5&26.4&8.8&33.5&38.5&47.6&54.3&34.1&45.7&33.4&35.7\\
& Resnet-50 & $641 \times 641$ &64.8&34.4&53.1&30.4&32.2&29.5&30.9&27.9&26.5&8.8&36.4&40.2&49.9&57.3&34.9&47.8&29.9&34.1\\
& Resnet-101 & $641 \times 641$ & 68.3&40.6&59.5&38.0&39.4&36.6&36.9&34.6&31.6&11.2&42.6&46.6&56.0&62.0&42.4&52.6&37.2&41.3 \\
\hline
\multirow{4}*{HrHRNet~\cite{higherhrnet}}
& \multirow{2}*{HrHRNet-W32}
& $512 \times 512$ & 67.1&39.9&59.4&34.2&37.0&35.2&35.1&32.5&34.0&12.5&43.3&47.6&54.9&60.6&43.4&50.3&42.2&35.0 \\
& & $640 \times 640$ &68.5&39.6&57.8&31.5&38.9&37.6&34.8&32.6&33.8&12.1&44.0&47.9&55.1&61.4&42.9&50.6&37.8&33.2\\
& \multirow{2}*{HrHRNet-W48}
&$512 \times 512$ & 68.5&41.9&61.2&39.3&42.2&40.1&36.1&33.5&35.1&13.1&45.0&49.1&56.7&62.1&45.5&51.4&42.2&36.9 \\
& &$640 \times 640$ &69.8&40.8&58.4&36.7&39.8&37.9&35.6&33.1&34.2&12.5&44.7&49.0&56.3&62.8&43.7&51.3&39.9&34.3 \\
\hline
\multicolumn{21}{c}{} \\[-5pt]
\multicolumn{21}{c}{\large Results of bottom-up methods on OCHuman-C} \\[5pt]
\hline
Method & Backbone & Input size & AP$^*$ & mPC & rPC & Gauss& Shot & Impulse & Defoucs & Glass & Motion & Zoom & Snow & Frost & Fog & Bright & Contrast & Elastic & Pixel & JPEG \\
\hline
\multirow{3}*{PifPaf~\cite{pifpaf}}
& ShuffleNet V2 & $641 \times 641$ & 37.8&30.8&81.5&28.7&28.8&28.1&32.7&32.2&32.9&17.2&26.3&30.5&36.4&36.1&30.5&36.2&32.8&32.6\\
& Resnet-50 & $641 \times 641$ & 37.6&30.8&82.0&30.1&30.0&29.4&33.3&31.6&32.4&16.5&27.0&30.5&36.5&36.5&29.9&36.1&31.8&30.7\\
& Resnet-101 & $641 \times 641$ & 39.6&34.9&88.1&35.6&35.6&35.3&37.2&35.3&36.3&20.2&31.4&34.4&39.2&38.8&34.8&38.9&35.5&34.6 \\
\hline
\multirow{4}*{HrHRNet~\cite{higherhrnet}}
& \multirow{2}*{HrHRNet-W32}
& $512 \times 512$ & 40.0&35.1&87.6&32.9&33.1&33.2&37.3&36.5&36.9&21.7&31.6&36.0&39.9&39.4&36.9&39.2&37.1&34.4 \\
& & $640 \times 640$ & 39.3&33.6&85.4&30.9&31.1&31.8&36.2&35.0&36.1&19.6&31.1&35.1&39.3&38.1&35.8&38.3&33.8&31.1\\
& \multirow{2}*{HrHRNet-W48}
& $512 \times 512$ &
41.7&36.7&87.9&35.5&35.6&35.7&38.8&37.5&38.5&22.0&33.5&37.6&41.5&40.5&39.0&40.6&38.0&35.6 \\
& & $640 \times 640$ & 40.9&35.4&86.4&33.0&33.3&33.5&37.7&36.6&37.6&21.1&32.4&37.4&41.5&39.9&37.2&39.9&36.6&32.6\\
\hline
    \end{tabular}}
    \label{tab:benchmark_all}
\end{table*}

\section{Related Work}

\begin{figure*}[th]
\centering
\includegraphics[width=1.0\textwidth]{figs/image_corruption.pdf}
\caption{\textbf{Visualization of examples in our benchmark datasets.} Our proposed benchmarks contain 15 different types of corruptions with different severity levels for a single clean image. The image corruption types are grouped into four main categories: noise, blur, weather, and digital. We sample one corruption type from each category in this figure.}
\label{corruption_images}
\end{figure*}

\subsection{Human Pose Estimation}
Human pose estimation (HPE) can be generally categorized into top-down and bottom-up methods. Top-down methods~\cite{CPM,stackhourglass,rmpe,hrnet,jin2020whole,liu2018cascaded, duan2019trb} divide the task into two stages: person detection and keypoint detection. SBL \cite{simple} presents a simple yet strong baseline network with several deconvolutional layers. HRNet \cite{hrnet} maintains high resolution representation combined with multi-scale feature fusions, and achieves the state-of-the-art performance on clean COCO dataset~\cite{coco}.
Bottom-up methods~\cite{openpose,asso_embedding,pifpaf,higherhrnet,jin2020differentiable,jin2019multi} first detect all the keypoints and then group them into person instances. PifPaf \cite{pifpaf} utilized a Part Intensity Field (PIF) to localize body parts and a Part Association Field (PAF)~\cite{openpose} to associate body parts to form full human poses. HigherHRNet \cite{higherhrnet} learns scale-aware representations using high-resolution feature pyramids, and groups keypoints with associative embeddings~\cite{asso_embedding}. In this paper, we establish the benchmark and extensively evaluate the robustness of these state-of-the-art top-down and bottom-up methods.

\subsection{Corruption Robustness}
Recent studies have explored the corruption robustness of image classification \cite{dodge2016understanding,imagenet_benchmark}, object detection \cite{det_benchmark}, and segmentation \cite{seg_benchmark}. In comparison, the task of HPE is more comprehensive, which requires a blend of classification and regression approaches to model human body structure. Data denoising~\cite{buades2005review} \textit{e}.\textit{g}.  sparse and redundant representations~\cite{elad2006image}, non-local algorithm~\cite{buades2005non}, and denoising auto-encoder~\cite{xie2012image} are effective in removing noise. However, such methods are noise-specific, thus are not applicable to improving robustness towards unforeseen noises. To improve general robustness, recent works have explored several useful techniques such as pre-training~\cite{pretrain}, stability training~\cite{stability_training}, stylized image~\cite{imagenet_texture}, NoisyStudent~\cite{noisystudent}, and histogram equalization~\cite{imagenet_benchmark}.

\subsection{Data Augmentation}
Data augmentation has been widely utilized as an effective
method to improve model generalization. However, improving the general model robustness to unseen image corruptions is difficult. According to \cite{vasiljevic2016examining,generalization}, augmenting with one specific type of noise enhances the performance on the target noise, but it does not generalize to other unseen distortions while degrading the performance on clean data. Information dropping methods~\cite{random_erase,hide_and_seek,cutout,gridmask} and multi-image mixing methods~\cite{mixup,cutmix,augmix} have gained decent improvements on clean data for image classification. Learned augmentation methods~\cite{adv_autoaug, autoaugment} have also been proposed to improve performance. For pose estimation, adversarial data augmentation methods~\cite{advpose_1, advpose_2} are leveraged to optimize augmentation hyper-parameters, \textit{e}.\textit{g}.  the rotation angle. However, they only focus on searching for common augmentation hyper-parameters and simply combine different augmentations sequentially to improve performance on clean data. Instead, we adversarially learn attention weights for mixing randomly augmented images to enhance model robustness. AugMix \cite{augmix} is proposed to mix augmented images using beta or dirichlet coefficients to boost the robustness on image classification. Instead of simply using a fixed augmentation sampler(\textit{e}.\textit{g}.  dirichlet distribution) to generate mixing weights, our AdvMix uses adversarial training to learn to generate appropriate mixing attention weights for each training sample.


\section{Introduction \& Motivations}\label{sec:intro}
\input{intro}

\section{Related Works}\label{sec:related}
\input{related}

\section{Adaptive Test-Time Augmentation}\label{sec:opt}
\input{adaptive_tta}

\section{Experimental Setup}\label{sec:setup}
\input{setup}

\section{Results}\label{sec:results}
\input{results}

\section{Conclusions}\label{sec:conc}
\input{conclusions}

\bibliographystyle{IEEEtran}

\subsection{Hardware Platform and Software Configurations}\label{sec:hw}
The hardware testbench is the Odroid-C2 platform powered with the Amlogic S905 SoC. The CPU is a quad-core ARM Cortex-A53 running @1.5GHz nominal frequency. The board runs Ubuntu Mate 18.04, kernel version 3.16.72-46, released by Hardkernel.
The inference engine is TensorFlow Lite 1.14; it offers a collection of neural-network routines optimized to run on the ARM Cortex-A architecture.
In our setup, TensorFlow Lite is cross-compiled using the GNU ARM Embedded Toolchain (version 6.5) \cite{linaro}.

\subsection{ConvNet Benchmarks}\label{sec:setup_convnets}
We adopted pre-trained ConvNets available on TensorFlow Hosted Models~\cite{tflitemodels} repository. Specifically, we considered a representative family of models for the mobile segment: {\em MobileNet}~\cite{howard2017mobilenets, sandler2018mobilenetv2}.
The networks are trained on the ImageNet dataset~\cite{deng2009imagenet} and quantized to 8-bit, a common choice for edge inference as it ensures lower memory footprint and faster processing speed with negligible accuracy loss with respect to floating-point.
For MobileNet family, we investigated two different versions of ConvNets listed in Table~\ref{tab:benchmarks}.
The column {\bf Storage} collects the size of the \texttt{tflite}, which contains the data structures needed to deploy the model on-chip, i.e., the network weights and the topology description. The column {\bf Top-1} refers to the top-1 classification accuracy measured on the ImageNet validation set without TTA, i.e., with a standard pre-processing consisting of resizing the images to 256$\times$256 pixels and extracting the central crop of size 224$\times$224. The column $\bm{L_\textrm{\textbf{nom}}}$ reports the nominal latency of a single inference at maximum performance (4-threads @1.5GHz).

The adopted benchmarks are state-of-the-art for image classification on embedded systems.
MobileNets are compact hand-crafted networks optimized for high performance and low memory.

\begin{table}[!t]
\centering
\caption{Memory, nominal top-1 accuracy without TTA (Top-1), and nominal latency ($L_\textrm{\textbf{nom}}$) of the selected benchmarks.}\label{tab:benchmarks}
\begin{tabular}{lcccc}
\toprule
{\bf ConvNet} & {\bf Storage} & {\bf Top-1} & $\bm{L_\textrm{\textbf{nom}}}$ \\
{} & {\bf {[}MB{]}} & {\bf [\%]} & {\bf {[}ms{]}} \\
\midrule
MobileNetV1 & 4.3    &  70.0   & $53.1$   \\
MobileNetV2  &3.4    &  70.8   & $44.2$   \\
\bottomrule
\end{tabular}
\vspace{-0.5cm}
\end{table}

\subsection{Augmentation Policies}\label{sec:setup_policy}
We adopted two popular TTA policies~\cite{krizhevsky2012imagenet, howard2013some, szegedy2015going}, which are described as follows:\\
{\bf 5-Crops (5C)} - starting from a $256\times256$ image, five crops of size $224\times224$ are extracted (the center crop and the four corner crops (top-left, top-right, bottom-left and bottom-right).\\
{\bf 10-Crops (10C)} - is an extension of the 5C policy; it applies the left-to-right horizontal flipping to the five crops of 5C for a total of 10 images.

These policies enable substantial improvements in the classification accuracy as reported  Table~\ref{tab:tta_accuracy}. Obviously, doubling the number of transformations (from 5C to 10C) increases the accuracy gain.  Moreover, the transformations get processed efficiently even on resource-constrained devices: on the Cortex-A53 CPU, cropping requires only \SI{0.8}{\milli\second} and horizontal flipping \SI{0.9}{\milli\second}.

\begin{table}[!h]
    \centering
    \caption{Accuracy gain (in \%) of 5-Crops (5C) and 10-Crops (10C) TTA policies compared to the nominal values for static TTA and AdapTTA ($\tau=0.8)$.}\label{tab:tta_accuracy}
    \begin{tabular}{lcc|cc}
    \toprule
    \multirow{3}{*}{\bf ConvNet} & \multicolumn{2}{c|}{\bf Static TTA} & \multicolumn{2}{c}{\bf AdapTTA}\\
    {\bf ConvNet} & {\bf 5C} & {\bf 10C} & {\bf 5C} & {\bf 10C}\\
    \midrule
    MobileNetV1  &2.7 &3.1 &2.7 &3.1\\
    MobileNetV2  &2.2 &2.9 &2.2 &2.9\\
    \bottomrule
    \end{tabular}
\vspace{-0.5cm}
\end{table}