$A = \dfrac{h}{(h^2 + l^2)\sqrt{h^2 + 2l^2}}$
$\dfrac{dA}{dh} = \dfrac{(h^2 + l^2)\sqrt{h^2 + 2l^2}(1) - h\,\left[ (h^2 + l^2)\dfrac{2h}{2\sqrt{h^2 + 2l^2}} + 2h \sqrt{h^2 + 2l^2} \right]}{(h^2 + l^2)^2(h^2 + 2l^2)} = 0$
$(h^2 + l^2)\sqrt{h^2 + 2l^2} - \dfrac{h^2(h^2 + l^2)}{\sqrt{h^2 + 2l^2}} - 2h \sqrt{h^2 + 2l^2} = 0$
$(h^2 + l^2)(h^2 + 2l^2) - h^2(h^2 + l^2) - 2h^2(h^2 + 2l^2) = 0$
$(h^4 + 3l^2h^2 + 2l^4) - (h^4 + l^2h^2) - (2h^4 + 4l^2h^2) = 0$
$-2h^4 - 2l^2h^2 + 2l^4 = 0$
$h^4 + l^2h^2 - l^4 = 0$
$h^2 = \dfrac{-l^2 \pm \sqrt{l^4 + 4l^4}}{2}$
$h^2 = \dfrac{-l^2 \pm \sqrt{5}\,l^2}{2}$
$h^2 = \dfrac{-1 \pm \sqrt{5}}{2}l^2$
$h = \sqrt{\dfrac{-1 - \sqrt{5}}{2}} \, l \,\, \text{ (imaginary)}$
$h = \sqrt{\dfrac{-1 + \sqrt{5}}{2}} \, l \,\, \text{ (ok!)}$
Use
$h = \sqrt{\dfrac{-1 + \sqrt{5}}{2}} \, l$
$h = 0.7862l$ answer