[This is a heuristic explanation.]

Ref:

• “Inference and Learning from Data” by Sayed.

• “A multivariate version of Stein’s identity” by Arnold, Castillo, Sarabia.

Lem [Stein Lemma]

Let ${ \mathbf{x} \in \mathbb{R} ^p }$ be a Gaussian random vector ${ \mathbf{x} \sim N(\overline{x}, R _x) }$ with ${ R _x > 0 . }$

Let ${ g : \mathbb{R} ^p \longrightarrow \mathbb{R} . }$

Let the expectations ${ \mathbb{E} \vert \partial g(\mathbf{x}) / \partial x _m \vert < \infty . }$

Then the expectation

\[{ \Large \boxed{ \mathbb{E} \left[ (\mathbf{x} - \overline{x}) g(\mathbf{x}) \right] = R _x \mathbb{E} \left[ \nabla _{x ^{T}} g(\mathbf{x}) \right] } }\]

where ${ \nabla _{x ^{T}} g(x) }$ is the column vector

\[{ \nabla _{x ^{T}} g(x) = \left( \frac{\partial g}{\partial x _1} (x), \ldots, \frac{\partial g}{\partial x _p}(x) \right) ^{T} .}\]

Intuition:

We will use integration by parts.

Note that the random vector ${ \mathbf{x} }$ can be replaced by

\[{ \mathbf{x} = \overline{x} + R _x ^{1/2} \mathbf{z}, \quad \text{ where } \mathbf{z} \sim N(0, I _p) . }\]

Hence

\[{ {\begin{aligned} &\, \mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ (\mathbf{x} - \overline{x}) g(\mathbf{x}) \right] \\ = &\, \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ (R _x ^{1/2} \mathbf{z}) g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right)\right] \\ = &\, R _x ^{1/2} \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] . \end{aligned}} }\]

Note that the component

\[{ {\begin{aligned} &\, \left( \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \right) _{i} \\ = &\, \int _{z _1} \ldots \int _{z _p} z _i g \left( \overline{x} + R _x ^{1/2} z \right) \frac{1}{(2 \pi) ^{p / 2}} e ^{- \frac{1}{2} z ^{T} z} \, dz _1 \ldots d z _p \\ = &\, \frac{1}{(2 \pi) ^{p/2}} \int _{z _i} z _i e ^{- \frac{1}{2} z _i ^2} \left( \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} g \left( \overline{x} + R _x ^{1/2} z \right) e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} \right) \, d z _i \end{aligned}} }\]

Defining

\[{ \varphi(z _i) := \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} g \left( \overline{x} + R _x ^{1/2} z \right) e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} }\]

we have

\[{ {\begin{aligned} &\, \left( \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \right) _{i} \\ = &\, \frac{1}{(2 \pi) ^{p/2}} \int _{z _i} z _i e ^{- \frac{1}{2} z _i ^2} \varphi(z _i) \, dz _i . \end{aligned}} }\]

Note that by integration by parts,

\[{ {\begin{aligned} &\, \left( \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \right) _{i} \\ = &\, \frac{1}{(2 \pi) ^{p/2}} \int _{z _i} z _i e ^{- \frac{1}{2} z _i ^2} \varphi(z _i) \, dz _i \\ = &\, \frac{1}{(2 \pi) ^{p/2}} \left( \left(- e ^{- \frac{1}{2} z _i ^2} \varphi(z _i) \right) \Big \vert _{- \infty} ^{+ \infty} + \int _{z _i} e ^{- \frac{1}{2} z _i ^2} \varphi ^{'} (z _i) \, d z _i \right) \end{aligned}} }\]

Note that intuitively, due to the rapid decay of ${ e ^{- \frac{1}{2} z _i ^2 }, }$

\[{ \text{Intuitively:} \quad \left(- e ^{- \frac{1}{2} z _i ^2} \varphi(z _i) \right) \Big \vert _{- \infty} ^{+ \infty} = 0. }\]

Hence

\[{ {\begin{aligned} &\, \left( \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \right) _{i} \\ = &\, \frac{1}{(2 \pi) ^{p/2}} \int _{z _i} e ^{- \frac{1}{2} z _i ^2} \varphi ^{'} (z _i) \, d z _i . \end{aligned}} }\]

Note that the derivative

\[{ {\begin{aligned} &\, \varphi ^{'} (z _i) \\ = &\, \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} \frac{\partial g \left( \overline{x} + R _x ^{1/2} z \right)}{\partial z _i} e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} \\ = &\, \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} \left(i ^{\text{th}} \text{ component of row } \nabla _{x} g \left( \overline{x} + R _x ^{1/2} z \right) R _x ^{1/2}\right) e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} \\ = &\, \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} \left(i ^{\text{th}} \text{ component of column } R _x ^{1/2} \nabla _{x ^{T}} g \left( \overline{x} + R _x ^{1/2} z \right) \right) e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} \\ = &\, \underbrace{\int _{z _1} \ldots \int _{z _p}} _{\text{skip } z _i} \left(R _x ^{1/2} \nabla _{x ^{T}} g \left( \overline{x} + R _x ^{1/2} z \right) \right) _i e ^{- \frac{1}{2} \left( \sum _{\nu \neq i} z _{\nu} ^2 \right)} \underbrace{d z _1 \ldots d z _p} _{\text{skip } dz _i} . \end{aligned}} }\]

Hence

\[{ {\begin{aligned} &\, \left( \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \right) _{i} \\ = &\, \frac{1}{(2 \pi) ^{p /2}} \int _{z _1} \ldots \int _{z _p} \left(R _x ^{1/2} \nabla _{x ^{T}} g \left( \overline{x} + R _x ^{1/2} z \right) \right) _i e ^{- \frac{1}{2} \left( \sum z _{\nu} ^2 \right)} \, dz _1 \ldots d z _p . \end{aligned}} }\]

Hence

\[{ {\begin{aligned} &\, \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \\ = &\, \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ R _x ^{1/2} \nabla _{x ^{T}} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \\ = &\, R _x ^{1/2} \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \nabla _{x ^{T}} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \\ = &\, R _x ^{1/2} \mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ \nabla _{x ^{T}} g (\mathbf{x}) \right]. \end{aligned}} }\]

Hence the original expression

\[{ {\begin{aligned} &\, \mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ (\mathbf{x} - \overline{x}) g(\mathbf{x}) \right] \\ = &\, R _x ^{1/2} \mathbb{E} _{\mathbf{z} \sim N(0, I _p)} \left[ \mathbf{z} g \left( \overline{x} + R _x ^{1/2} \mathbf{z} \right) \right] \\ = &\, R _x \mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ \nabla _{x ^{T}} g (\mathbf{x}) \right] . \end{aligned}} }\]

That is,

\[{ \boxed{\mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ (\mathbf{x} - \overline{x}) g(\mathbf{x}) \right] = R _x \mathbb{E} _{\mathbf{x} \sim N(\overline{x}, R _x)} \left[ \nabla _{x ^{T}} g (\mathbf{x}) \right] } }\]

as needed. ${ \blacksquare }$