// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com)
// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#ifndef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
#define EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H

// If using dynamic dispatch, set the CPU target.
#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
#pragma GCC push_options
#pragma GCC target("cpu=power10,htm")
#endif

#ifdef __has_builtin
#if !__has_builtin(__builtin_vsx_assemble_pair)
#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
#endif
#if !__has_builtin(__builtin_vsx_disassemble_pair)
#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
#endif
#endif

// IWYU pragma: private
#include "../../InternalHeaderCheck.h"

#include "MatrixProductMMAbfloat16.h"

namespace Eigen {

namespace internal {

#define accColsC (accCols / 2)

EIGEN_ALWAYS_INLINE void bsetzeroMMA(__vector_quad* acc)
{
  __builtin_mma_xxsetaccz(acc);
}

template<typename DataMapper, typename Packet, bool full>
EIGEN_ALWAYS_INLINE void storeAccumulator(Index i, const DataMapper& data, const Packet& alpha, const Index elements, __vector_quad* acc)
{
  PacketBlock<Packet, 4> result;
  __builtin_mma_disassemble_acc(&result.packet, acc);

  PacketBlock<Packet, 4> tRes;
  if (full) {
    EIGEN_UNUSED_VARIABLE(elements);
    bload<DataMapper, Packet, 0, ColMajor, false, 4>(tRes, data, i, 0);
    bscale<Packet, 4>(tRes, result, alpha);
    bstore<DataMapper, Packet, 4>(tRes, data, i);
  } else {
    bload_partial<DataMapper, Packet, 0, false, 4>(tRes, data, i, elements);
    bscale<Packet, 4>(tRes, result, alpha);
    bstore_partial<DataMapper, Packet, 4>(tRes, data, i, elements);
  }
}

template<typename DataMapper, typename Packet, typename Packetc, const Index accCols, const Index accCols2>
EIGEN_ALWAYS_INLINE void storeComplexAccumulator(Index i, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal, __vector_quad* accImag)
{
  constexpr bool full = (accCols2 > accColsC);
  PacketBlock<Packet, 4> resultReal, resultImag;
  __builtin_mma_disassemble_acc(&resultReal.packet, accReal);
  __builtin_mma_disassemble_acc(&resultImag.packet, accImag);

  PacketBlock<Packetc, 8> tRes;
  bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0);

  PacketBlock<Packet, 4> taccReal, taccImag;
  bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask);

  PacketBlock<Packetc, 4> acc1, acc2;
  bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2);

  bstore<DataMapper, Packetc, 4>(acc1, data, i);
  if (full) {
    bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC);
  }
}

// Defaults to float32, since Eigen still supports C++03 we can't use default template arguments
template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b)
{
  if(NegativeAccumulate)
  {
    __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
  } else {
    __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
  }
}

template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b)
{
  if(NegativeAccumulate)
  {
    __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
  } else {
    __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
  }
}

template<typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, Packet& lhsVi, const RhsPacket& rhsV, RhsPacket& rhsVi)
{
  pgerMMA<Packet, RhsPacket, false>(accReal,  rhsV,  lhsV);
  if(LhsIsReal) {
    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi,  lhsV);
    EIGEN_UNUSED_VARIABLE(lhsVi);
  } else {
    if(!RhsIsReal) {
      pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi,  lhsV);
    } else {
      EIGEN_UNUSED_VARIABLE(rhsVi);
    }
    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag,  rhsV, lhsVi);
  }
}

// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
template<typename Packet>
EIGEN_ALWAYS_INLINE Packet ploadRhs(const __UNPACK_TYPE__(Packet)* rhs)
{
  return ploadu<Packet>(rhs);
}

template<typename Scalar, typename Packet>
EIGEN_ALWAYS_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV)
{
  rhsV = ploadRhs<Packet>(rhs);
} 

template<>
EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double* rhs, __vector_pair& rhsV)
{
#if EIGEN_COMP_LLVM
  __builtin_vsx_assemble_pair(&rhsV,
    reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs + (sizeof(Packet2d) / sizeof(double)))),
    reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs)));
#else
  rhsV = *reinterpret_cast<__vector_pair *>(const_cast<double *>(rhs));
#endif
}

EIGEN_ALWAYS_INLINE void ploadLhsMMA(const double* lhs, __vector_pair& lhsV)
{
  ploadRhsMMA(lhs, lhsV);
}

#define GEMM_MULTIPLE_COLS

// Disable in GCC until unnecessary register moves are fixed
//#if (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
#if EIGEN_COMP_LLVM
#define VECTOR_PAIR_LOADS_LHS
#endif

// PEEL_MMA loop factor.
#ifdef GEMM_MULTIPLE_COLS
#define PEEL_MMA 8
#else
// Register spillage with GCC12+
#if EIGEN_COMP_LLVM || (__GNUC__ < 12) || defined(VECTOR_PAIR_LOADS_LHS)
#define PEEL_MMA 7
#else
#define PEEL_MMA 6
#endif
#endif

#define MICRO_MMA_UNROLL(func) \
  func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)

#define MICRO_MMA_WORK(func, type, peel) \
  if (accItr == 1) { \
    func(0,type,peel,0,0) func(1,type,peel,1,0) func(2,type,peel,2,0) func(3,type,peel,3,0) \
    func(4,type,peel,4,0) func(5,type,peel,5,0) func(6,type,peel,6,0) func(7,type,peel,7,0) \
  } else if (accItr == 2) { \
    func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,1,0) func(3,type,peel,1,1) \
    func(4,type,peel,2,0) func(5,type,peel,2,1) func(6,type,peel,3,0) func(7,type,peel,3,1) \
  } else { \
    func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,0,2) func(3,type,peel,0,3) \
    func(4,type,peel,1,0) func(5,type,peel,1,1) func(6,type,peel,1,2) func(7,type,peel,1,3) \
  }

#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right) \
  if (unroll_factor > left) { \
    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV##left); \
  }

#ifdef VECTOR_PAIR_LOADS_LHS
#define MICRO_MMA_WORK_TWO(iter, type, peel, left, right) \
  if (unroll_factor > left) { \
    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV2##left.packet[peel & 1]); \
  }

#define MICRO_MMA_LOAD1_TWO(lhs_ptr, left) \
  if (unroll_factor > left) { \
    if (MICRO_NORMAL(left)) { \
      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr##left), plhsV##left); \
      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsV2##left.packet), &plhsV##left); \
      lhs_ptr##left += accCols*2; \
    } else { \
      lhsV2##left.packet[0] = ploadLhs<Packet>(lhs_ptr##left); \
      lhsV2##left.packet[1] = ploadLhs<Packet>(lhs_ptr##left + accCols2); \
      lhs_ptr##left += accCols2*2; \
      EIGEN_UNUSED_VARIABLE(plhsV##left); \
    } \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhsV2##left); \
    EIGEN_UNUSED_VARIABLE(plhsV##left); \
  }

#define MICRO_MMA_LOAD_TWO(left) MICRO_MMA_LOAD1_TWO(lhs_ptr, left)
#endif

#define MICRO_MMA_UNROLL_ITER(func, val) \
  func(val,0) \
  if (accItr > 1) { \
    func(val,1) \
    if (accItr > 2) { \
      func(val,2) \
      func(val,3) \
    } \
  }

#define MICRO_MMA_LOAD_ONE_RHS1(peel, right) \
  ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);

#define MICRO_MMA_LOAD_ONE_RHS(peel) \
  MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)

#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel) \
  if (PEEL_MMA > peel) { \
    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
    MICRO_MMA_LOAD_ONE_RHS(peel) \
    MICRO_MMA_UNROLL(funcl) \
    MICRO_MMA_WORK(funcw, type, peel) \
  }

#ifndef VECTOR_PAIR_LOADS_LHS
#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type) \
  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
  MICRO_MMA_TYPE_PEEL(funcw,funcl,type,0) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,1) \
  MICRO_MMA_TYPE_PEEL(funcw,funcl,type,2) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,3) \
  MICRO_MMA_TYPE_PEEL(funcw,funcl,type,4) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,5) \
  MICRO_MMA_TYPE_PEEL(funcw,funcl,type,6) MICRO_MMA_TYPE_PEEL(funcw,funcl,type,7)
#else
#define MICRO_MMA_LOAD_TWO_RHS(peel1, right) \
  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr##right + (accRows * peel1)), prhsV##peel1); \
  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1);

#define MICRO_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \
  if (PEEL_MMA > peel2) { \
    PacketBlock<Packet,2> lhsV20, lhsV21, lhsV22, lhsV23, lhsV24, lhsV25, lhsV26, lhsV27; \
    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3, plhsV4, plhsV5, plhsV6, plhsV7; \
    if (sizeof(type) == 16) { \
      MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_TWO_RHS, peel1) \
    } else { \
      EIGEN_UNUSED_VARIABLE(prhsV##peel1); \
      MICRO_MMA_LOAD_ONE_RHS(peel1) \
      MICRO_MMA_LOAD_ONE_RHS(peel2) \
    } \
    MICRO_MMA_UNROLL(funcl2) \
    MICRO_MMA_WORK(funcw2, type, peel1) \
    MICRO_MMA_WORK(funcw2, type, peel2) \
  } else { \
    EIGEN_UNUSED_VARIABLE(prhsV##peel1); \
    MICRO_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1) \
  }

#define MICRO_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type) \
  type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
  __vector_pair prhsV0, prhsV2, prhsV4, prhsV6; \
  MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,0,1) \
  MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,2,3) \
  MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,4,5) \
  MICRO_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,6,7)
#endif

#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \
  type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1]; \
  MICRO_MMA_TYPE_PEEL(funcw,funcl,type,0)

#define MICRO_MMA_UPDATE_RHS1(size, right) \
  rhs_ptr##right += (accRows * size);

#define MICRO_MMA_UPDATE_RHS(size) \
  MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)

#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size) \
  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \
  MICRO_MMA_UPDATE_RHS(size)

#ifndef VECTOR_PAIR_LOADS_LHS
#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)
#else
#define MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_TYPE, size) \
  MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, MICRO_MMA_WORK_TWO, MICRO_MMA_LOAD_TWO, RhsPacket) \
  MICRO_MMA_UPDATE_RHS(size)

#define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE2(MICRO_MMA_UNROLL_TYPE_PEEL2, PEEL_MMA)
#endif

#define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)

#define MICRO_MMA_DST_PTR_ONE(iter) \
  if (unroll_factor * accItr > iter) { \
    bsetzeroMMA(&accZero##iter); \
  } else { \
    EIGEN_UNUSED_VARIABLE(accZero##iter); \
  }

#define MICRO_MMA_DST_PTR MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)

#define MICRO_MMA_SRC_PTR MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)

#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)

#define MICRO_MMA_STORE_ONE(iter, left, right) \
  if (unroll_factor > left) { \
    storeAccumulator<DataMapper, Packet, MICRO_NORMAL_PARTIAL(left)>(row + left*accCols, res##right, pAlpha, accCols2, &accZero##iter); \
  }

#define MICRO_MMA_ITER_UNROLL(func) \
  if (accItr == 1) { \
    func(0,0,0) func(1,1,0) func(2,2,0) func(3,3,0) \
    func(4,4,0) func(5,5,0) func(6,6,0) func(7,7,0) \
  } else if (accItr == 2) { \
    func(0,0,0) func(1,0,1) func(2,1,0) func(3,1,1) \
    func(4,2,0) func(5,2,1) func(6,3,0) func(7,3,1) \
  } else { \
    func(0,0,0) func(1,0,1) func(2,0,2) func(3,0,3) \
    func(4,1,0) func(5,1,1) func(6,1,2) func(7,1,3) \
  }

#define MICRO_MMA_STORE MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)

#define MICRO_MMA_EXTRA_ROWS(right) \
  gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(res3##right, blockA, rhs_base + right*accRows*strideB, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlpha, pMask);

#define MICRO_MMA_EXTRA_ROWS1(val, right) \
  MICRO_MMA_EXTRA_ROWS(right);

template<int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool full, const Index accItr>
EIGEN_ALWAYS_INLINE void gemm_unrolled_MMA_iteration(
  const DataMapper& res0,
  const DataMapper& res1,
  const DataMapper& res2,
  const DataMapper& res3,
  const Scalar* lhs_base,
  const Scalar* rhs_base,
  Index depth,
  Index strideA,
  Index strideB,
  Index offsetA,
  Index& row,
  const Packet& pAlpha,
  Index accCols2
  )
{
  const Scalar* rhs_ptr0 = rhs_base, * rhs_ptr1 = NULL, * rhs_ptr2 = NULL, * rhs_ptr3 = NULL;
  const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL;
  __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;

  if (accItr > 1) {
    rhs_ptr1 = rhs_base + (accRows * strideB);
  } else {
    EIGEN_UNUSED_VARIABLE(strideB);
    EIGEN_UNUSED_VARIABLE(rhs_ptr1);
    EIGEN_UNUSED_VARIABLE(res1);
  }
  if (accItr > 2) {
    rhs_ptr2 = rhs_base + (2 * accRows * strideB);
    rhs_ptr3 = rhs_base + (3 * accRows * strideB);
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr3);
    EIGEN_UNUSED_VARIABLE(res2);
    EIGEN_UNUSED_VARIABLE(res3);
  }

  MICRO_MMA_SRC_PTR
  MICRO_MMA_DST_PTR

  Index k = 0, depth2 = depth - PEEL_MMA;
  for(; k <= depth2; k += PEEL_MMA)
  {
    EIGEN_POWER_PREFETCH(rhs_ptr);
    MICRO_MMA_PREFETCH
    MICRO_MMA_ONE_PEEL
  }
  for(; k < depth; k++)
  {
    MICRO_MMA_ONE
  }
  MICRO_MMA_STORE

  MICRO_UPDATE
}

#define MICRO_MMA_UNROLL_ITER2(N, M) \
  gemm_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, RhsPacket, DataMapper, accRows, accCols, !M, accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha, M ? remaining_rows : accCols); \
  if (M) return;

#define MICRO_MMA_ROWS(n) \
  while(row + n*accCols <= rows) { \
    MICRO_MMA_UNROLL_ITER2(n, 0); \
  }

template<typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, const Index accItr>
EIGEN_ALWAYS_INLINE void gemmMMA_cols(
  const DataMapper& res,
  const Scalar* blockA,
  const Scalar* blockB,
  Index depth,
  Index strideA,
  Index offsetA,
  Index strideB,
  Index offsetB,
  Index col,
  Index rows,
  Index remaining_rows,
  const Packet& pAlpha,
  const Packet& pMask)
{
  const DataMapper res30 = res.getSubMapper(0, col);
  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows*1) : res30;
  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows*2) : res30;
  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows*3) : res30;

  const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB;
  const Scalar* lhs_base = blockA + accCols*offsetA;
  Index row = 0;

#define MAX_MMA_UNROLL 7

#if MAX_MMA_UNROLL < 2
  if (1) {
#elif MAX_MMA_UNROLL < 4
  if (accItr <= 2) {
#else
  if (accItr == 1) {
#endif
    MICRO_MMA_ROWS(MAX_MMA_UNROLL);
  } else if (accItr == 2) {
    MICRO_MMA_ROWS(4);
  } else {
    MICRO_MMA_ROWS(2);
  }
  switch( (rows-row)/accCols ) {
#if MAX_MMA_UNROLL > 7
    case 7:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 6
    case 6:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 5
    case 5:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 4
    case 4:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 3
    case 3:
      if (accItr <= 2) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 2
    case 2:
      if (accItr <= 2) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 1
    case 1:
      MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1)
      break;
#endif
    default:
      break;
  }
#undef MAX_MMA_UNROLL

  if(remaining_rows > 0)
  {
    MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0)
  }
}

#define MICRO_MMA_COLS(n) \
  for(; col + n*accRows <= cols; col += n*accRows) \
  { \
    gemmMMA_cols<Scalar, Packet, RhsPacket2, DataMapper, accRows, accCols, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \
  }

template<typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols>
void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
{
      const Index remaining_rows = rows % accCols;

      if( strideA == -1 ) strideA = depth;
      if( strideB == -1 ) strideB = depth;

      const Packet pAlpha = pset1<Packet>(alpha);
      const Packet pMask  = bmask<Packet>(remaining_rows);

      typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;

      Index col = 0;
#ifdef GEMM_MULTIPLE_COLS
      MICRO_MMA_COLS(4);
      MICRO_MMA_COLS(2);
#endif
      MICRO_MMA_COLS(1);

      if (col != cols)
      {
        gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlpha, pMask);
      }
}

#define advanceRows ((LhsIsReal) ? 1 : 2)
#define advanceCols ((RhsIsReal) ? 1 : 2)

// PEEL_COMPLEX_MMA loop factor.
#ifdef GEMM_MULTIPLE_COLS
#define PEEL_COMPLEX_MMA 4
#else
#define PEEL_COMPLEX_MMA 3
#endif

#define MICRO_COMPLEX_MMA_UNROLL(func) \
  func(0) func(1) func(2) func(3)

#define MICRO_COMPLEX_MMA_WORK(func, type, peel) \
  if (accItr == 1) { \
    func(0,type,peel,0,0) func(1,type,peel,1,0) func(2,type,peel,2,0) func(3,type,peel,3,0) \
  } else if (accItr == 2) { \
    func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,1,0) func(3,type,peel,1,1) \
  } else { \
    func(0,type,peel,0,0) func(1,type,peel,0,1) func(2,type,peel,0,2) func(3,type,peel,0,3) \
  }

#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right) \
  if (unroll_factor > left) { \
    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \
  }

#ifdef VECTOR_PAIR_LOADS_LHS
#define MICRO_COMPLEX_MMA_WORK_TWO(iter, type, peel, left, right) \
  if (unroll_factor > left) { \
    pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV2##left.packet[peel & 1], lhsVi2##left.packet[peel & 1], rhsV##right[peel], rhsVi##right[peel]); \
  }

#define MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left) \
  if (!LhsIsReal && (unroll_factor > left)) { \
    if (MICRO_NORMAL(left)) { \
      ploadLhsMMA(reinterpret_cast<const double*>(lhs_ptr_real##left + imag_delta), plhsVi##left); \
      __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&lhsVi2##left.packet), &plhsVi##left); \
    } else { \
      lhsVi2##left.packet[0] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2); \
      lhsVi2##left.packet[1] = ploadLhs<Packet>(lhs_ptr_real##left + imag_delta2 + accCols2); \
      EIGEN_UNUSED_VARIABLE(plhsVi##left); \
    } \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhsVi2##left); \
    EIGEN_UNUSED_VARIABLE(plhsVi##left); \
  } \
  MICRO_MMA_LOAD1_TWO(lhs_ptr_real, left)

#define MICRO_COMPLEX_MMA_LOAD_TWO(left) MICRO_COMPLEX_MMA_LOAD1_TWO(lhs_ptr, left)
#endif

#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right) \
  ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]); \
  if (!RhsIsReal) { \
    ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \
  }

#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) \
  MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)

#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel) \
  if (PEEL_COMPLEX_MMA > peel) { \
    Packet lhsV0, lhsV1, lhsV2, lhsV3; \
    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \
    MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) \
    MICRO_COMPLEX_MMA_UNROLL(funcl) \
    MICRO_COMPLEX_MMA_WORK(funcw, type, peel) \
  }

#ifndef VECTOR_PAIR_LOADS_LHS
#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type) \
  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1]; \
  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,0) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,1) \
  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,3)
#else
#define MICRO_COMPLEX_MMA_LOAD_TWO_RHS(peel1, right) \
  ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_real##right + (accRows * peel1)), prhsV##peel1); \
  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsV##right[peel1]), &prhsV##peel1); \
  if(!RhsIsReal) { \
    ploadRhsMMA(reinterpret_cast<const double*>(rhs_ptr_imag##right + (accRows * peel1)), prhsVi##peel1); \
    __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(&rhsVi##right[peel1]), &prhsVi##peel1); \
  } else { \
    EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \
  }

#define MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type, peel1, peel2) \
  if (PEEL_COMPLEX_MMA > peel2) { \
    PacketBlock<Packet,2> lhsV20, lhsV21, lhsV22, lhsV23; \
    PacketBlock<Packet,2> lhsVi20, lhsVi21, lhsVi22, lhsVi23; \
    __vector_pair plhsV0, plhsV1, plhsV2, plhsV3; \
    __vector_pair plhsVi0, plhsVi1, plhsVi2, plhsVi3; \
    if (sizeof(type) == 16) { \
      MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_TWO_RHS, peel1) \
    } else { \
      EIGEN_UNUSED_VARIABLE(prhsV##peel1); \
      EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \
      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel1); \
      MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel2); \
    } \
    MICRO_COMPLEX_MMA_UNROLL(funcl2) \
    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel1) \
    MICRO_COMPLEX_MMA_WORK(funcw2, type, peel2) \
  } else { \
    EIGEN_UNUSED_VARIABLE(prhsV##peel1); \
    EIGEN_UNUSED_VARIABLE(prhsVi##peel1); \
    MICRO_COMPLEX_MMA_TYPE_PEEL(funcw1, funcl1, type, peel1) \
  }

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2(funcw1, funcl1, funcw2, funcl2, type) \
  type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1]; \
  __vector_pair prhsV0, prhsV2; \
  __vector_pair prhsVi0, prhsVi2; \
  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,0,1) \
  MICRO_COMPLEX_MMA_TYPE_PEEL2(funcw1,funcl1,funcw2,funcl2,type,2,3)
#endif

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type) \
  type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \
  MICRO_COMPLEX_MMA_TYPE_PEEL(funcw,funcl,type,0)

#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right) \
  rhs_ptr_real##right += (accRows * size); \
  if(!RhsIsReal) rhs_ptr_imag##right += (accRows * size);

#define MICRO_COMPLEX_MMA_UPDATE_RHS(size) \
  MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)

#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size) \
  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \
  MICRO_COMPLEX_MMA_UPDATE_RHS(size);

#ifndef VECTOR_PAIR_LOADS_LHS
#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)
#else
#define MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_TYPE, size) \
  MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, MICRO_COMPLEX_MMA_WORK_TWO, MICRO_COMPLEX_MMA_LOAD_TWO, RhsPacket) \
  MICRO_COMPLEX_MMA_UPDATE_RHS(size);

#define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE2(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL2, PEEL_COMPLEX_MMA)
#endif

#define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)

#define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \
  if (unroll_factor * accItr > iter) { \
    bsetzeroMMA(&accReal##iter); \
    bsetzeroMMA(&accImag##iter); \
  } else { \
    EIGEN_UNUSED_VARIABLE(accReal##iter); \
    EIGEN_UNUSED_VARIABLE(accImag##iter); \
  }

#define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)

#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)

#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)

#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right) \
  if (unroll_factor > left) { \
    storeComplexAccumulator<DataMapper, Packet, Packetc, accCols, (unroll_factor != (left + 1)) ? accCols : accCols2>(row + left*accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter); \
  }

#define MICRO_COMPLEX_MMA_ITER_UNROLL(func) \
  if (accItr == 1) { \
    func(0,0,0) func(1,1,0) func(2,2,0) func(3,3,0) \
  } else if (accItr == 2) { \
    func(0,0,0) func(1,0,1) func(2,1,0) func(3,1,1) \
  } else { \
    func(0,0,0) func(1,0,1) func(2,0,2) func(3,0,3) \
  }

#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)

#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right) \
  gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res3##right, blockA, rhs_base + right*accRows*(RhsIsReal ? 1 : 2)*strideB, depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask);

#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right) \
  MICRO_COMPLEX_MMA_EXTRA_ROWS(right);

template<int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration(
  const DataMapper& res0,
  const DataMapper& res1,
  const DataMapper& res2,
  const DataMapper& res3,
  const Scalar* lhs_base,
  const Scalar* rhs_base,
  Index depth,
  Index strideA,
  Index offsetA,
  Index strideB,
  Index& row,
  const Packet& pAlphaReal,
  const Packet& pAlphaImag,
  const Packet& pMask)
{
  const Scalar* rhs_ptr_real0 = rhs_base, * rhs_ptr_real1 = NULL, * rhs_ptr_real2 = NULL, * rhs_ptr_real3 = NULL;
  const Scalar* rhs_ptr_imag0 = NULL, * rhs_ptr_imag1 = NULL, * rhs_ptr_imag2 = NULL, * rhs_ptr_imag3 = NULL;
  const Index imag_delta = accCols*strideA;
  const Index imag_delta2 = accCols2*strideA;

  if(!RhsIsReal) {
    rhs_ptr_imag0 = rhs_base + accRows*strideB;
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0);
  }
  if (accItr > 1) {
    if(!RhsIsReal) {
      rhs_ptr_real1 = rhs_base + (2*accRows*strideB);
      rhs_ptr_imag1 = rhs_base + (3*accRows*strideB);
    } else {
      rhs_ptr_real1 = rhs_base + accRows*strideB;
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
    }
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real1);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
    EIGEN_UNUSED_VARIABLE(res1);
  }
  if (accItr > 2) {
    if(!RhsIsReal) {
      rhs_ptr_real2 = rhs_base + (4*accRows*strideB);
      rhs_ptr_imag2 = rhs_base + (5*accRows*strideB);
      rhs_ptr_real3 = rhs_base + (6*accRows*strideB);
      rhs_ptr_imag3 = rhs_base + (7*accRows*strideB);
    } else {
      rhs_ptr_real2 = rhs_base + (2*accRows*strideB);
      rhs_ptr_real3 = rhs_base + (3*accRows*strideB);
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
    }
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real3);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
    EIGEN_UNUSED_VARIABLE(res2);
    EIGEN_UNUSED_VARIABLE(res3);
  }
  const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_real1 = NULL;
  const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_real3 = NULL;
  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;

  MICRO_COMPLEX_MMA_SRC_PTR
  MICRO_COMPLEX_MMA_DST_PTR

  Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA;
  for(; k <= depth2; k += PEEL_COMPLEX_MMA)
  {
    EIGEN_POWER_PREFETCH(rhs_ptr_real);
    if(!RhsIsReal) {
      EIGEN_POWER_PREFETCH(rhs_ptr_imag);
    }
    MICRO_COMPLEX_MMA_PREFETCH
    MICRO_COMPLEX_MMA_ONE_PEEL
  }
  for(; k < depth; k++)
  {
    MICRO_COMPLEX_MMA_ONE
  }
  MICRO_COMPLEX_MMA_STORE

  MICRO_COMPLEX_UPDATE
}

#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M) \
  gemm_complex_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \
  if (M) return;

#define MICRO_COMPLEX_MMA_ROWS(n) \
  while(row + n*accCols <= rows) { \
    MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \
  }

template<typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void gemmMMA_complex_cols(
  const DataMapper& res,
  const Scalar* blockA,
  const Scalar* blockB,
  Index depth,
  Index strideA,
  Index offsetA,
  Index strideB,
  Index offsetB,
  Index col,
  Index rows,
  Index remaining_rows,
  const Packet& pAlphaReal,
  const Packet& pAlphaImag,
  const Packet& pMask)
{
  const DataMapper res30 = res.getSubMapper(0, col);
  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows*1) : res30;
  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows*2) : res30;
  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows*3) : res30;

  const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB;
  const Scalar* lhs_base = blockA + accCols*offsetA;
  Index row = 0;

#define MAX_COMPLEX_MMA_UNROLL 4

#if MAX_COMPLEX_MMA_UNROLL < 2
  if (1) {
#elif MAX_COMPLEX_MMA_UNROLL < 4
  if (accItr <= 2) {
#else
  if (accItr == 1) {
#endif
    MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL);
  } else if (accItr == 2) {
    MICRO_COMPLEX_MMA_ROWS(2);
  } else {
    MICRO_COMPLEX_MMA_ROWS(1);
  }
  switch( (rows-row)/accCols ) {
#if MAX_COMPLEX_MMA_UNROLL > 3
    case 3:
      if (accItr == 1) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3)
      }
      break;
#endif
#if MAX_COMPLEX_MMA_UNROLL > 2
    case 2:
      if (accItr == 1) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2)
      }
      break;
#endif
#if MAX_COMPLEX_MMA_UNROLL > 1
    case 1:
      if (accItr <= 2) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1)
      }
      break;
#endif
    default:
      break;
  }
#undef MAX_COMPLEX_MMA_UNROLL

  if(remaining_rows > 0)
  {
    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0)
  }
}

#define MICRO_COMPLEX_MMA_COLS(n) \
  for(; col + n*accRows <= cols; col += n*accRows) \
  { \
    gemmMMA_complex_cols<Scalar, Packet, Packetc, RhsPacket2, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlphaReal, pAlphaImag, pMask); \
  }

template<typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
{
      const Index remaining_rows = rows % accCols;

      if( strideA == -1 ) strideA = depth;
      if( strideB == -1 ) strideB = depth;

      const Packet pAlphaReal = pset1<Packet>(alpha.real());
      const Packet pAlphaImag = pset1<Packet>(alpha.imag());
      const Packet pMask = bmask<Packet>(remaining_rows);

      const Scalar* blockA = (Scalar *) blockAc;
      const Scalar* blockB = (Scalar *) blockBc;

      typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;

      Index col = 0;
#ifdef GEMM_MULTIPLE_COLS
      MICRO_COMPLEX_MMA_COLS(4);
      MICRO_COMPLEX_MMA_COLS(2);
#endif
      MICRO_COMPLEX_MMA_COLS(1);

      if (col != cols)
      {
        gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask);
      }
}

#undef accColsC
#undef advanceRows
#undef advanceCols

} // end namespace internal

} // end namespace Eigen

#if defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
#pragma GCC pop_options
#endif

#endif // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H

