\documentclass[a4paper,12pt]{article}
\newcommand{\ds}{\displaystyle}
\newcommand{\un}{\underline}
\newcommand{\undb}{\underbrace}
\newcommand{\pl}{\partial}
\parindent=0pt
\begin{document}

{\bf Question}

Find the region in which the equation

$$[(x-y)^2-1]u_{xx}+2u{xy}+[(x-y)^2-1]u_{yy}=0$$

is of hyperbolic type. Write down the differential equation of the
characteristics and, by making the change of variables $\xi=x+y,\
\eta=x-y$, show that it may be written as

$$\ds\frac{d\xi}{d\eta}=\pm\ds\frac{eta}{\sqrt{2-\eta^2}}.$$

Deduce the form of the characteristics.


\medskip

{\bf Answer}

$a=[(x-y)^2-1],\ b=1,\ c=[(x-y)^2-1]$

$b^2-ac=1-[(x-y)^2-1]^2=(x-y)^2(2-(x-y)^2)$

so is hyperbolic for $0<(x-y)^2<2$

\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ i.e., $0<|x-y|<\sqrt{2}$

Parabolic on $x=y$ i.e., $o=|x-y|,\ \sqrt{2}=|x-y|,\
(x-y)=\sqrt{2}$

elliptic for $(x-y)^2.2$ i.e., $\sqrt{2}<|x-y|$

Characteristics when parabolic/hyperbolic given by

$$[(x-y)^2-1]dx^2-2dxdy+[(x-y)^2-1]dy^2=0\ \ \ (A)$$

or $\un{\ds\frac{dy}{dx}=\ds\frac{2 \pm
\sqrt{(x-y)^2(2-(x-y)^2)}}{[(x-y)^2-1]}}$ \un{not nice to solve}

Make change  $\left\{\begin{array} {l} \xi = x+y\\ \eta=x-y
\end{array} \right\}$ as per question

So

$\left\{\begin{array} {l} x = \ds\frac{1}{2}(\xi+\eta)\\
y=\ds\frac{!}{2}9\xi-\eta) \end{array} \right\} \Rightarrow
\left\{\begin{array} {l} dx = \ds\frac{1}{2}(d\xi+d\eta)\\
dy=\ds\frac{1}{2}(d\xi-d\eta) \end{array} \right\}$

Substitute this into (A):

$\ds\frac{(\eta^2-1)}{4}(d\xi+d\eta)^2-\ds\frac{1}
{2(d\xi^2-d\eta^2)}+\ds\frac{(\eta^2-1)}{4}(d\xi-d\eta)^2=0$

$\Rightarrow [(\eta-1)^2-1]d\xi^2+[(\eta^2-1)+1]d\eta^2=0$

$\Rightarrow
\ds\frac{d\xi}{d\eta}=\pm\sqrt{-\left[\ds\frac{(\eta^2-1)+1}{(\eta^2-1)-1}\right]}=\pm
\ds\frac{\eta}{(2-\eta^2)^{\frac{1}{2}}}$ as required

\newpage
Solve this for $\eta$

\begin{eqnarray*} \ds\int d\xi & = & \pm \ds\int\ds\frac{\,d\eta
\eta}{(2-\eta^2)^{\frac{1}{2}}}\\ \xi & = &
\mp\sqrt{2-\eta^2}+const \end{eqnarray*}

or $$\un{(\xi+c)^2+\eta^2=2}$$

Circles centred at $\xi=-c$ radius $\sqrt{2}$

In original coords

$$(x+y+c)^2+(x-y)^2=2$$

or see diagram below:

PICTURE  \vspace{2in}

\end{document}