Don't return Matrix3 from Matrix4 member functions.

Matrix3 is for 2D affine transformations, while Matrix4 is for 3D.
Returning Matrix3 would allow doing this, which isn't meaningful
operation at all:

    Matrix4 transformation;
    Vector2 wtf = transformation.rotationScaling().translation();

src/Math/Matrix3.h

@@ -25,7 +25,7 @@
 namespace Magnum { namespace Math {
 
 /**
-@brief 3x3 matrix
+@brief 3x3 matrix for affine transformations in 2D
 @tparam T   Data type
 
 Provides functions for transformations in 2D. See Matrix4 for 3D

src/Math/Matrix4.h

@@ -25,7 +25,7 @@
 namespace Magnum { namespace Math {
 
 /**
-@brief 4x4 matrix
+@brief 4x4 matrix for affine transformations in 3D
 @tparam T   Data type
 
 Provides functions for transformations in 3D. See Matrix3 for 2D
@@ -138,8 +138,9 @@ template<class T> class Matrix4: public Matrix<4, T> {
          * @see rotation() const, rotation(T, const Vector3&),
          *      Matrix3::rotationScaling() const
          */
-        inline Matrix3<T> rotationScaling() const {
+        inline Matrix<3, T> rotationScaling() const {
-            return Matrix3<T>::from(
+            /* Not Matrix3, because it is for affine 2D transformations */
+            return Matrix<3, T>::from(
                 (*this)[0].xyz(),
                 (*this)[1].xyz(),
                 (*this)[2].xyz());
@@ -152,8 +153,9 @@ template<class T> class Matrix4: public Matrix<4, T> {
          * @see rotationScaling() const, rotation(T, const Vector3&),
          *      Matrix3::rotation() const
          */
-        inline Matrix3<T> rotation() const {
+        inline Matrix<3, T> rotation() const {
-            return Matrix3<T>::from(
+            /* Not Matrix3, because it is for affine 2D transformations */
+            return Matrix<3, T>::from(
                 (*this)[0].xyz().normalized(),
                 (*this)[1].xyz().normalized(),
                 (*this)[2].xyz().normalized());
@@ -175,8 +177,6 @@ template<class T> class Matrix4: public Matrix<4, T> {
         }
 
         #ifndef DOXYGEN_GENERATING_OUTPUT
-        inline Matrix3<T> ij(size_t skipRow, size_t skipCol) const { return Matrix<4, T>::ij(skipRow, skipCol); }
-
         inline Point3D<T> operator*(const Point3D<T>& other) const {
             return Matrix<4, T>::operator*(other);
         }