<commit_before>/*
	mvsas.c - Marvell 88SE6440 SAS/SATA support

	Copyright 2007 Red Hat, Inc.
	Copyright 2008 Marvell. <<EMAIL>>

	This program is free software; you can redistribute it and/or
	modify it under the terms of the GNU General Public License as
	published by the Free Software Foundation; either version 2,
	or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty
	of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	See the GNU General Public License for more details.

	You should have received a copy of the GNU General Public
	License along with this program; see the file COPYING.	If not,
	write to the Free Software Foundation, 675 Mass Ave, Cambridge,
	MA 02139, USA.

	---------------------------------------------------------------

	Random notes:
	* hardware supports controlling the endian-ness of data
	  structures.  this permits elimination of all the le32_to_cpu()
	  and cpu_to_le32() conversions.

 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/ctype.h>
#include <scsi/libsas.h>
#include <scsi/scsi_tcq.h>
#include <scsi/sas_ata.h>
#include <asm/io.h>

#define DRV_NAME	"mvsas"
#define DRV_VERSION	"0.5.2"
#define _MV_DUMP	0
#define MVS_DISABLE_NVRAM
#define MVS_DISABLE_MSI

#define mr32(reg)	readl(regs + MVS_##reg)
#define mw32(reg,val)	writel((val), regs + MVS_##reg)
#define mw32_f(reg,val)	do {			\
	writel((val), regs + MVS_##reg);	\
	readl(regs + MVS_##reg);		\
	} while (0)

#define MVS_ID_NOT_MAPPED	0x7f
#define MVS_CHIP_SLOT_SZ	(1U << mvi->chip->slot_width)

/* offset for D2H FIS in the Received FIS List Structure */
#define SATA_RECEIVED_D2H_FIS(reg_set)	\
	((void *) mvi->rx_fis + 0x400 + 0x100 * reg_set + 0x40)
#define SATA_RECEIVED_PIO_FIS(reg_set)	\
	((void *) mvi->rx_fis + 0x400 + 0x100 * reg_set + 0x20)
#define UNASSOC_D2H_FIS(id)		\
	((void *) mvi->rx_fis + 0x100 * id)

#define for_each_phy(__lseq_mask, __mc, __lseq, __rest)			\
	for ((__mc) = (__lseq_mask), (__lseq) = 0;			\
					(__mc) != 0 && __rest;		\
					(++__lseq), (__mc) >>= 1)

/* driver compile-time configuration */
enum driver_configuration {
	MVS_TX_RING_SZ		= 1024,	/* TX ring size (12-bit) */
	MVS_RX_RING_SZ		= 1024, /* RX ring size (12-bit) */
					/* software requires power-of-2
					   ring size */

	MVS_SLOTS		= 512,	/* command slots */
	MVS_SLOT_BUF_SZ		= 8192, /* cmd tbl + IU + status + PRD */
	MVS_SSP_CMD_SZ		= 64,	/* SSP command table buffer size */
	MVS_ATA_CMD_SZ		= 96,	/* SATA command table buffer size */
	MVS_OAF_SZ		= 64,	/* Open address frame buffer size */

	MVS_RX_FIS_COUNT	= 17,	/* Optional rx'd FISs (max 17) */

	MVS_QUEUE_SIZE		= 30,	/* Support Queue depth */
	MVS_CAN_QUEUE		= MVS_SLOTS - 1,	/* SCSI Queue depth */
};

/* unchangeable hardware details */
enum hardware_details {
	MVS_MAX_PHYS		= 8,	/* max. possible phys */
	MVS_MAX_PORTS		= 8,	/* max. possible ports */
	MVS_RX_FISL_SZ		= 0x400 + (MVS_RX_FIS_COUNT * 0x100),
};

/* peripheral registers (BAR2) */
enum peripheral_registers {
	SPI_CTL			= 0x10,	/* EEPROM control */
	SPI_CMD			= 0x14,	/* EEPROM command */
	SPI_DATA		= 0x18, /* EEPROM data */
};

enum peripheral_register_bits {
	TWSI_RDY		= (1U << 7),	/* EEPROM interface ready */
	TWSI_RD			= (1U << 4),	/* EEPROM read access */

	SPI_ADDR_MASK		= 0x3ffff,	/* bits 17:0 */
};

/* enhanced mode registers (BAR4) */
enum hw_registers {
	MVS_GBL_CTL		= 0x04,  /* global control */
	MVS_GBL_INT_STAT	= 0x08,  /* global irq status */
	MVS_GBL_PI		= 0x0C,  /* ports implemented bitmask */
	MVS_GBL_PORT_TYPE	= 0xa0,  /* port type */

	MVS_CTL			= 0x100, /* SAS/SATA port configuration */
	MVS_PCS			= 0x104, /* SAS/SATA port control/status */
	MVS_CMD_LIST_LO		= 0x108, /* cmd list addr */
	MVS_CMD_LIST_HI		= 0x10C,
	MVS_RX_FIS_LO		= 0x110, /* RX FIS list addr */
	MVS_RX_FIS_HI		= 0x114,

	MVS_TX_CFG		= 0x120, /* TX configuration */
	MVS_TX_LO		= 0x124, /* TX (delivery) ring addr */
	MVS_TX_HI		= 0x128,

	MVS_TX_PROD_IDX		= 0x12C, /* TX producer pointer */
	MVS_TX_CONS_IDX		= 0x130, /* TX consumer pointer (RO) */
	MVS_RX_CFG		= 0x134, /* RX configuration */
	MVS_RX_LO		= 0x138, /* RX (completion) ring addr */
	MVS_RX_HI		= 0x13C,
	MVS_RX_CONS_IDX		= 0x140, /* RX consumer pointer (RO) */

	MVS_INT_COAL		= 0x148, /* Int coalescing config */
	MVS_INT_COAL_TMOUT	= 0x14C, /* Int coalescing timeout */
	MVS_INT_STAT		= 0x150, /* Central int status */
	MVS_INT_MASK		= 0x154, /* Central int enable */
	MVS_INT_STAT_SRS	= 0x158, /* SATA register set status */
	MVS_INT_MASK_SRS	= 0x15C,

					 /* ports 1-3 follow after this */
	MVS_P0_INT_STAT		= 0x160, /* port0 interrupt status */
	MVS_P0_INT_MASK		= 0x164, /* port0 interrupt mask */
	MVS_P4_INT_STAT		= 0x200, /* Port 4 interrupt status */
	MVS_P4_INT_MASK		= 0x204, /* Port 4 interrupt enable mask */

					 /* ports 1-3 follow after this */
	MVS_P0_SER_CTLSTAT	= 0x180, /* port0 serial control/status */
	MVS_P4_SER_CTLSTAT	= 0x220, /* port4 serial control/status */

	MVS_CMD_ADDR		= 0x1B8, /* Command register port (addr) */
	MVS_CMD_DATA		= 0x1BC, /* Command register port (data) */

					 /* ports 1-3 follow after this */
	MVS_P0_CFG_ADDR		= 0x1C0, /* port0 phy register address */
	MVS_P0_CFG_DATA		= 0x1C4, /* port0 phy register data */
	MVS_P4_CFG_ADDR		= 0x230, /* Port 4 config address */
	MVS_P4_CFG_DATA		= 0x234, /* Port 4 config data */

					 /* ports 1-3 follow after this */
	MVS_P0_VSR_ADDR		= 0x1E0, /* port0 VSR address */
	MVS_P0_VSR_DATA		= 0x1E4, /* port0 VSR data */
	MVS_P4_VSR_ADDR		= 0x250, /* port 4 VSR addr */
	MVS_P4_VSR_DATA		= 0x254, /* port 4 VSR data */
};

enum hw_register_bits {
	/* MVS_GBL_CTL */
	INT_EN			= (1U << 1),	/* Global int enable */
	HBA_RST			= (1U << 0),	/* HBA reset */

	/* MVS_GBL_INT_STAT */
	INT_XOR			= (1U << 4),	/* XOR engine event */
	INT_SAS_SATA		= (1U << 0),	/* SAS/SATA event */

	/* MVS_GBL_PORT_TYPE */			/* shl for ports 1-3 */
	SATA_TARGET		= (1U << 16),	/* port0 SATA target enable */
	MODE_AUTO_DET_PORT7 = (1U << 15),	/* port0 SAS/SATA autodetect */
	MODE_AUTO_DET_PORT6 = (1U << 14),
	MODE_AUTO_DET_PORT5 = (1U << 13),
	MODE_AUTO_DET_PORT4 = (1U << 12),
	MODE_AUTO_DET_PORT3 = (1U << 11),
	MODE_AUTO_DET_PORT2 = (1U << 10),
	MODE_AUTO_DET_PORT1 = (1U << 9),
	MODE_AUTO_DET_PORT0 = (1U << 8),
	MODE_AUTO_DET_EN    =	MODE_AUTO_DET_PORT0 | MODE_AUTO_DET_PORT1 |
				MODE_AUTO_DET_PORT2 | MODE_AUTO_DET_PORT3 |
				MODE_AUTO_DET_PORT4 | MODE_AUTO_DET_PORT5 |
				MODE_AUTO_DET_PORT6 | MODE_AUTO_DET_PORT7,
	MODE_SAS_PORT7_MASK = (1U << 7),  /* port0 SAS(1), SATA(0) mode */
	MODE_SAS_PORT6_MASK = (1U << 6),
	MODE_SAS_PORT5_MASK = (1U << 5),
	MODE_SAS_PORT4_MASK = (1U << 4),
	MODE_SAS_PORT3_MASK = (1U << 3),
	MODE_SAS_PORT2_MASK = (1U << 2),
	MODE_SAS_PORT1_MASK = (1U << 1),
	MODE_SAS_PORT0_MASK = (1U << 0),
	MODE_SAS_SATA	=	MODE_SAS_PORT0_MASK | MODE_SAS_PORT1_MASK |
				MODE_SAS_PORT2_MASK | MODE_SAS_PORT3_MASK |
				MODE_SAS_PORT4_MASK | MODE_SAS_PORT5_MASK |
				MODE_SAS_PORT6_MASK | MODE_SAS_PORT7_MASK,

				/* SAS_MODE value may be
				 * dictated (in hw) by values
				 * of SATA_TARGET & AUTO_DET
				 */

	/* MVS_TX_CFG */
	TX_EN			= (1U << 16),	/* Enable TX */
	TX_RING_SZ_MASK		= 0xfff,	/* TX ring size, bits 11:0 */

	/* MVS_RX_CFG */
	RX_EN			= (1U << 16),	/* Enable RX */
	RX_RING_SZ_MASK		= 0xfff,	/* RX ring size, bits 11:0 */

	/* MVS_INT_COAL */
	COAL_EN			= (1U << 16),	/* Enable int coalescing */

	/* MVS_INT_STAT, MVS_INT_MASK */
	CINT_I2C		= (1U << 31),	/* I2C event */
	CINT_SW0		= (1U << 30),	/* software event 0 */
	CINT_SW1		= (1U << 29),	/* software event 1 */
	CINT_PRD_BC		= (1U << 28),	/* PRD BC err for read cmd */
	CINT_DMA_PCIE		= (1U << 27),	/* DMA to PCIE timeout */
	CINT_MEM		= (1U << 26),	/* int mem parity err */
	CINT_I2C_SLAVE		= (1U << 25),	/* slave I2C event */
	CINT_SRS		= (1U << 3),	/* SRS event */
	CINT_CI_STOP		= (1U << 1),	/* cmd issue stopped */
	CINT_DONE		= (1U << 0),	/* cmd completion */

						/* shl for ports 1-3 */
	CINT_PORT_STOPPED	= (1U << 16),	/* port0 stopped */
	CINT_PORT		= (1U << 8),	/* port0 event */
	CINT_PORT_MASK_OFFSET	= 8,
	CINT_PORT_MASK		= (0xFF << CINT_PORT_MASK_OFFSET),

	/* TX (delivery) ring bits */
	TXQ_CMD_SHIFT		= 29,
	TXQ_CMD_SSP		= 1,		/* SSP protocol */
	TXQ_CMD_SMP		= 2,		/* SMP protocol */
	TXQ_CMD_STP		= 3,		/* STP/SATA protocol */
	TXQ_CMD_SSP_FREE_LIST	= 4,		/* add to SSP targ free list */
	TXQ_CMD_SLOT_RESET	= 7,		/* reset command slot */
	TXQ_MODE_I		= (1U << 28),	/* mode: 0=target,1=initiator */
	TXQ_PRIO_HI		= (1U << 27),	/* priority: 0=normal, 1=high */
	TXQ_SRS_SHIFT		= 20,		/* SATA register set */
	TXQ_SRS_MASK		= 0x7f,
	TXQ_PHY_SHIFT		= 12,		/* PHY bitmap */
	TXQ_PHY_MASK		= 0xff,
	TXQ_SLOT_MASK		= 0xfff,	/* slot number */

	/* RX (completion) ring bits */
	RXQ_GOOD		= (1U << 23),	/* Response good */
	RXQ_SLOT_RESET		= (1U << 21),	/* Slot reset complete */
	RXQ_CMD_RX		= (1U << 20),	/* target cmd received */
	RXQ_ATTN		= (1U << 19),	/* attention */
	RXQ_RSP			= (1U << 18),	/* response frame xfer'd */
	RXQ_ERR			= (1U << 17),	/* err info rec xfer'd */
	RXQ_DONE		= (1U << 16),	/* cmd complete */
	RXQ_SLOT_MASK		= 0xfff,	/* slot number */

	/* mvs_cmd_hdr bits */
	MCH_PRD_LEN_SHIFT	= 16,		/* 16-bit PRD table len */
	MCH_SSP_FR_TYPE_SHIFT	= 13,		/* SSP frame type */

						/* SSP initiator only */
	MCH_SSP_FR_CMD		= 0x0,		/* COMMAND frame */

						/* SSP initiator or target */
	MCH_SSP_FR_TASK		= 0x1,		/* TASK frame */

						/* SSP target only */
	MCH_SSP_FR_XFER_RDY	= 0x4,		/* XFER_RDY frame */
	MCH_SSP_FR_RESP		= 0x5,		/* RESPONSE frame */
	MCH_SSP_FR_READ		= 0x6,		/* Read DATA frame(s) */
	MCH_SSP_FR_READ_RESP	= 0x7,		/* ditto, plus RESPONSE */

	MCH_PASSTHRU		= (1U << 12),	/* pass-through (SSP) */
	MCH_FBURST		= (1U << 11),	/* first burst (SSP) */
	MCH_CHK_LEN		= (1U << 10),	/* chk xfer len (SSP) */
	MCH_RETRY		= (1U << 9),	/* tport layer retry (SSP) */
	MCH_PROTECTION		= (1U << 8),	/* protection info rec (SSP) */
	MCH_RESET		= (1U << 7),	/* Reset (STP/SATA) */
	MCH_FPDMA		= (1U << 6),	/* First party DMA (STP/SATA) */
	MCH_ATAPI		= (1U << 5),	/* ATAPI (STP/SATA) */
	MCH_BIST		= (1U << 4),	/* BIST activate (STP/SATA) */
	MCH_PMP_MASK		= 0xf,		/* PMP from cmd FIS (STP/SATA)*/

	CCTL_RST		= (1U << 5),	/* port logic reset */

						/* 0(LSB first), 1(MSB first) */
	CCTL_ENDIAN_DATA	= (1U << 3),	/* PRD data */
	CCTL_ENDIAN_RSP		= (1U << 2),	/* response frame */
	CCTL_ENDIAN_OPEN	= (1U << 1),	/* open address frame */
	CCTL_ENDIAN_CMD		= (1U << 0),	/* command table */

	/* MVS_Px_SER_CTLSTAT (per-phy control) */
	PHY_SSP_RST		= (1U << 3),	/* reset SSP link layer */
	PHY_BCAST_CHG		= (1U << 2),	/* broadcast(change) notif */
	PHY_RST_HARD		= (1U << 1),	/* hard reset + phy reset */
	PHY_RST			= (1U << 0),	/* phy reset */
	PHY_MIN_SPP_PHYS_LINK_RATE_MASK = (0xF << 8),
	PHY_MAX_SPP_PHYS_LINK_RATE_MASK = (0xF << 12),
	PHY_NEG_SPP_PHYS_LINK_RATE_MASK_OFFSET = (16),
	PHY_NEG_SPP_PHYS_LINK_RATE_MASK =
			(0xF << PHY_NEG_SPP_PHYS_LINK_RATE_MASK_OFFSET),
	PHY_READY_MASK		= (1U << 20),

	/* MVS_Px_INT_STAT, MVS_Px_INT_MASK (per-phy events) */
	PHYEV_DEC_ERR		= (1U << 24),	/* Phy Decoding Error */
	PHYEV_UNASSOC_FIS	= (1U << 19),	/* unassociated FIS rx'd */
	PHYEV_AN		= (1U << 18),	/* SATA async notification */
	PHYEV_BIST_ACT		= (1U << 17),	/* BIST activate FIS */
	PHYEV_SIG_FIS		= (1U << 16),	/* signature FIS */
	PHYEV_POOF		= (1U << 12),	/* phy ready from 1 -> 0 */
	PHYEV_IU_BIG		= (1U << 11),	/* IU too long err */
	PHYEV_IU_SMALL		= (1U << 10),	/* IU too short err */
	PHYEV_UNK_TAG		= (1U << 9),	/* unknown tag */
	PHYEV_BROAD_CH		= (1U << 8),	/* broadcast(CHANGE) */
	PHYEV_COMWAKE		= (1U << 7),	/* COMWAKE rx'd */
	PHYEV_PORT_SEL		= (1U << 6),	/* port selector present */
	PHYEV_HARD_RST		= (1U << 5),	/* hard reset rx'd */
	PHYEV_ID_TMOUT		= (1U << 4),	/* identify timeout */
	PHYEV_ID_FAIL		= (1U << 3),	/* identify failed */
	PHYEV_ID_DONE		= (1U << 2),	/* identify done */
	PHYEV_HARD_RST_DONE	= (1U << 1),	/* hard reset done */
	PHYEV_RDY_CH		= (1U << 0),	/* phy ready changed state */

	/* MVS_PCS */
	PCS_EN_SATA_REG_SHIFT	= (16),		/* Enable SATA Register Set */
	PCS_EN_PORT_XMT_SHIFT	= (12),		/* Enable Port Transmit */
	PCS_EN_PORT_XMT_SHIFT2	= (8),		/* For 6480 */
	PCS_SATA_RETRY		= (1U << 8),	/* retry ctl FIS on R_ERR */
	PCS_RSP_RX_EN		= (1U << 7),	/* raw response rx */
	PCS_SELF_CLEAR		= (1U << 5),	/* self-clearing int mode */
	PCS_FIS_RX_EN		= (1U << 4),	/* FIS rx enable */
	PCS_CMD_STOP_ERR	= (1U << 3),	/* cmd stop-on-err enable */
	PCS_CMD_RST		= (1U << 1),	/* reset cmd issue */
	PCS_CMD_EN		= (1U << 0),	/* enable cmd issue */

	/* Port n Attached Device Info */
	PORT_DEV_SSP_TRGT	= (1U << 19),
	PORT_DEV_SMP_TRGT	= (1U << 18),
	PORT_DEV_STP_TRGT	= (1U << 17),
	PORT_DEV_SSP_INIT	= (1U << 11),
	PORT_DEV_SMP_INIT	= (1U << 10),
	PORT_DEV_STP_INIT	= (1U << 9),
	PORT_PHY_ID_MASK	= (0xFFU << 24),
	PORT_DEV_TRGT_MASK	= (0x7U << 17),
	PORT_DEV_INIT_MASK	= (0x7U << 9),
	PORT_DEV_TYPE_MASK	= (0x7U << 0),

	/* Port n PHY Status */
	PHY_RDY			= (1U << 2),
	PHY_DW_SYNC		= (1U << 1),
	PHY_OOB_DTCTD		= (1U << 0),

	/* VSR */
	/* PHYMODE 6 (CDB) */
	PHY_MODE6_LATECLK	= (1U << 29),	/* Lock Clock */
	PHY_MODE6_DTL_SPEED	= (1U << 27),	/* Digital Loop Speed */
	PHY_MODE6_FC_ORDER	= (1U << 26),	/* Fibre Channel Mode Order*/
	PHY_MODE6_MUCNT_EN	= (1U << 24),	/* u Count Enable */
	PHY_MODE6_SEL_MUCNT_LEN	= (1U << 22),	/* Training Length Select */
	PHY_MODE6_SELMUPI	= (1U << 20),	/* Phase Multi Select (init) */
	PHY_MODE6_SELMUPF	= (1U << 18),	/* Phase Multi Select (final) */
	PHY_MODE6_SELMUFF	= (1U << 16),	/* Freq Loop Multi Sel(final) */
	PHY_MODE6_SELMUFI	= (1U << 14),	/* Freq Loop Multi Sel(init) */
	PHY_MODE6_FREEZE_LOOP	= (1U << 12),	/* Freeze Rx CDR Loop */
	PHY_MODE6_INT_RXFOFFS	= (1U << 3),	/* Rx CDR Freq Loop Enable */
	PHY_MODE6_FRC_RXFOFFS	= (1U << 2),	/* Initial Rx CDR Offset */
	PHY_MODE6_STAU_0D8	= (1U << 1),	/* Rx CDR Freq Loop Saturate */
	PHY_MODE6_RXSAT_DIS	= (1U << 0),	/* Saturate Ctl */
};

enum mvs_info_flags {
	MVF_MSI			= (1U << 0),	/* MSI is enabled */
	MVF_PHY_PWR_FIX		= (1U << 1),	/* bug workaround */
};

enum sas_cmd_port_registers {
	CMD_CMRST_OOB_DET	= 0x100, /* COMRESET OOB detect register */
	CMD_CMWK_OOB_DET	= 0x104, /* COMWAKE OOB detect register */
	CMD_CMSAS_OOB_DET	= 0x108, /* COMSAS OOB detect register */
	CMD_BRST_OOB_DET	= 0x10c, /* burst OOB detect register */
	CMD_OOB_SPACE		= 0x110, /* OOB space control register */
	CMD_OOB_BURST		= 0x114, /* OOB burst control register */
	CMD_PHY_TIMER		= 0x118, /* PHY timer control register */
	CMD_PHY_CONFIG0		= 0x11c, /* PHY config register 0 */
	CMD_PHY_CONFIG1		= 0x120, /* PHY config register 1 */
	CMD_SAS_CTL0		= 0x124, /* SAS control register 0 */
	CMD_SAS_CTL1		= 0x128, /* SAS control register 1 */
	CMD_SAS_CTL2		= 0x12c, /* SAS control register 2 */
	CMD_SAS_CTL3		= 0x130, /* SAS control register 3 */
	CMD_ID_TEST		= 0x134, /* ID test register */
	CMD_PL_TIMER		= 0x138, /* PL timer register */
	CMD_WD_TIMER		= 0x13c, /* WD timer register */
	CMD_PORT_SEL_COUNT	= 0x140, /* port selector count register */
	CMD_APP_MEM_CTL		= 0x144, /* Application Memory Control */
	CMD_XOR_MEM_CTL		= 0x148, /* XOR Block Memory Control */
	CMD_DMA_MEM_CTL		= 0x14c, /* DMA Block Memory Control */
	CMD_PORT_MEM_CTL0	= 0x150, /* Port Memory Control 0 */
	CMD_PORT_MEM_CTL1	= 0x154, /* Port Memory Control 1 */
	CMD_SATA_PORT_MEM_CTL0	= 0x158, /* SATA Port Memory Control 0 */
	CMD_SATA_PORT_MEM_CTL1	= 0x15c, /* SATA Port Memory Control 1 */
	CMD_XOR_MEM_BIST_CTL	= 0x160, /* XOR Memory BIST Control */
	CMD_XOR_MEM_BIST_STAT	= 0x164, /* XOR Memroy BIST Status */
	CMD_DMA_MEM_BIST_CTL	= 0x168, /* DMA Memory BIST Control */
	CMD_DMA_MEM_BIST_STAT	= 0x16c, /* DMA Memory BIST Status */
	CMD_PORT_MEM_BIST_CTL	= 0x170, /* Port Memory BIST Control */
	CMD_PORT_MEM_BIST_STAT0 = 0x174, /* Port Memory BIST Status 0 */
	CMD_PORT_MEM_BIST_STAT1 = 0x178, /* Port Memory BIST Status 1 */
	CMD_STP_MEM_BIST_CTL	= 0x17c, /* STP Memory BIST Control */
	CMD_STP_MEM_BIST_STAT0	= 0x180, /* STP Memory BIST Status 0 */
	CMD_STP_MEM_BIST_STAT1	= 0x184, /* STP Memory BIST Status 1 */
	CMD_RESET_COUNT		= 0x188, /* Reset Count */
	CMD_MONTR_DATA_SEL	= 0x18C, /* Monitor Data/Select */
	CMD_PLL_PHY_CONFIG	= 0x190, /* PLL/PHY Configuration */
	CMD_PHY_CTL		= 0x194, /* PHY Control and Status */
	CMD_PHY_TEST_COUNT0	= 0x198, /* Phy Test Count 0 */
	CMD_PHY_TEST_COUNT1	= 0x19C, /* Phy Test Count 1 */
	CMD_PHY_TEST_COUNT2	= 0x1A0, /* Phy Test Count 2 */
	CMD_APP_ERR_CONFIG	= 0x1A4, /* Application Error Configuration */
	CMD_PND_FIFO_CTL0	= 0x1A8, /* Pending FIFO Control 0 */
	CMD_HOST_CTL		= 0x1AC, /* Host Control Status */
	CMD_HOST_WR_DATA	= 0x1B0, /* Host Write Data */
	CMD_HOST_RD_DATA	= 0x1B4, /* Host Read Data */
	CMD_PHY_MODE_21		= 0x1B8, /* Phy Mode 21 */
	CMD_SL_MODE0		= 0x1BC, /* SL Mode 0 */
	CMD_SL_MODE1		= 0x1C0, /* SL Mode 1 */
	CMD_PND_FIFO_CTL1	= 0x1C4, /* Pending FIFO Control 1 */
};

/* SAS/SATA configuration port registers, aka phy registers */
enum sas_sata_config_port_regs {
	PHYR_IDENTIFY		= 0x00,	/* info for IDENTIFY frame */
	PHYR_ADDR_LO		= 0x04,	/* my SAS address (low) */
	PHYR_ADDR_HI		= 0x08,	/* my SAS address (high) */
	PHYR_ATT_DEV_INFO	= 0x0C,	/* attached device info */
	PHYR_ATT_ADDR_LO	= 0x10,	/* attached dev SAS addr (low) */
	PHYR_ATT_ADDR_HI	= 0x14,	/* attached dev SAS addr (high) */
	PHYR_SATA_CTL		= 0x18,	/* SATA control */
	PHYR_PHY_STAT		= 0x1C,	/* PHY status */
	PHYR_SATA_SIG0		= 0x20,	/*port SATA signature FIS(Byte 0-3) */
	PHYR_SATA_SIG1		= 0x24,	/*port SATA signature FIS(Byte 4-7) */
	PHYR_SATA_SIG2		= 0x28,	/*port SATA signature FIS(Byte 8-11) */
	PHYR_SATA_SIG3		= 0x2c,	/*port SATA signature FIS(Byte 12-15) */
	PHYR_R_ERR_COUNT	= 0x30, /* port R_ERR count register */
	PHYR_CRC_ERR_COUNT	= 0x34, /* port CRC error count register */
	PHYR_WIDE_PORT		= 0x38,	/* wide port participating */
	PHYR_CURRENT0		= 0x80,	/* current connection info 0 */
	PHYR_CURRENT1		= 0x84,	/* current connection info 1 */
	PHYR_CURRENT2		= 0x88,	/* current connection info 2 */
};

/*  SAS/SATA Vendor Specific Port Registers */
enum sas_sata_vsp_regs {
	VSR_PHY_STAT		= 0x00, /* Phy Status */
	VSR_PHY_MODE1		= 0x01, /* phy tx */
	VSR_PHY_MODE2		= 0x02, /* tx scc */
	VSR_PHY_MODE3		= 0x03, /* pll */
	VSR_PHY_MODE4		= 0x04, /* VCO */
	VSR_PHY_MODE5		= 0x05, /* Rx */
	VSR_PHY_MODE6		= 0x06, /* CDR */
	VSR_PHY_MODE7		= 0x07, /* Impedance */
	VSR_PHY_MODE8		= 0x08, /* Voltage */
	VSR_PHY_MODE9		= 0x09, /* Test */
	VSR_PHY_MODE10		= 0x0A, /* Power */
	VSR_PHY_MODE11		= 0x0B, /* Phy Mode */
	VSR_PHY_VS0		= 0x0C, /* Vednor Specific 0 */
	VSR_PHY_VS1		= 0x0D, /* Vednor Specific 1 */
};

enum pci_cfg_registers {
	PCR_PHY_CTL	= 0x40,
	PCR_PHY_CTL2	= 0x90,
	PCR_DEV_CTRL	= 0xE8,
};

enum pci_cfg_register_bits {
	PCTL_PWR_ON	= (0xFU << 24),
	PCTL_OFF	= (0xFU << 12),
	PRD_REQ_SIZE	= (0x4000),
	PRD_REQ_MASK	= (0x00007000),
};

enum nvram_layout_offsets {
	NVR_SIG		= 0x00,		/* 0xAA, 0x55 */
	NVR_SAS_ADDR	= 0x02,		/* 8-byte SAS address */
};

enum chip_flavors {
	chip_6320,
	chip_6440,
	chip_6480,
};

enum port_type {
	PORT_TYPE_SAS	=  (1L << 1),
	PORT_TYPE_SATA	=  (1L << 0),
};

/* Command Table Format */
enum ct_format {
	/* SSP */
	SSP_F_H		=  0x00,
	SSP_F_IU	=  0x18,
	SSP_F_MAX	=  0x4D,
	/* STP */
	STP_CMD_FIS	=  0x00,
	STP_ATAPI_CMD	=  0x40,
	STP_F_MAX	=  0x10,
	/* SMP */
	SMP_F_T		=  0x00,
	SMP_F_DEP	=  0x01,
	SMP_F_MAX	=  0x101,
};

enum status_buffer {
	SB_EIR_OFF	=  0x00,	/* Error Information Record */
	SB_RFB_OFF	=  0x08,	/* Response Frame Buffer */
	SB_RFB_MAX	=  0x400,	/* RFB size*/
};

enum error_info_rec {
	CMD_ISS_STPD	= (1U << 31),	/* Cmd Issue Stopped */
	CMD_PI_ERR	= (1U << 30),	/* Protection info error.  see flags2 */
	RSP_OVER	= (1U << 29),	/* rsp buffer overflow */
	RETRY_LIM	= (1U << 28),	/* FIS/frame retry limit exceeded */
	UNK_FIS 	= (1U << 27),	/* unknown FIS */
	DMA_TERM	= (1U << 26),	/* DMA terminate primitive rx'd */
	SYNC_ERR	= (1U << 25),	/* SYNC rx'd during frame xmit */
	TFILE_ERR	= (1U << 24),	/* SATA taskfile Error bit set */
	R_ERR		= (1U << 23),	/* SATA returned R_ERR prim */
	RD_OFS		= (1U << 20),	/* Read DATA frame invalid offset */
	XFER_RDY_OFS	= (1U << 19),	/* XFER_RDY offset error */
	UNEXP_XFER_RDY	= (1U << 18),	/* unexpected XFER_RDY error */
	DATA_OVER_UNDER = (1U << 16),	/* data overflow/underflow */
	INTERLOCK	= (1U << 15),	/* interlock error */
	NAK		= (1U << 14),	/* NAK rx'd */
	ACK_NAK_TO	= (1U << 13),	/* ACK/NAK timeout */
	CXN_CLOSED	= (1U << 12),	/* cxn closed w/out ack/nak */
	OPEN_TO 	= (1U << 11),	/* I_T nexus lost, open cxn timeout */
	PATH_BLOCKED	= (1U << 10),	/* I_T nexus lost, pathway blocked */
	NO_DEST 	= (1U << 9),	/* I_T nexus lost, no destination */
	STP_RES_BSY	= (1U << 8),	/* STP resources busy */
	BREAK		= (1U << 7),	/* break received */
	BAD_DEST	= (1U << 6),	/* bad destination */
	BAD_PROTO	= (1U << 5),	/* protocol not supported */
	BAD_RATE	= (1U << 4),	/* cxn rate not supported */
	WRONG_DEST	= (1U << 3),	/* wrong destination error */
	CREDIT_TO	= (1U << 2),	/* credit timeout */
	WDOG_TO 	= (1U << 1),	/* watchdog timeout */
	BUF_PAR 	= (1U << 0),	/* buffer parity error */
};

enum error_info_rec_2 {
	SLOT_BSY_ERR	= (1U << 31),	/* Slot Busy Error */
	GRD_CHK_ERR	= (1U << 14),	/* Guard Check Error */
	APP_CHK_ERR	= (1U << 13),	/* Application Check error */
	REF_CHK_ERR	= (1U << 12),	/* Reference Check Error */
	USR_BLK_NM	= (1U << 0),	/* User Block Number */
};

struct mvs_chip_info {
	u32		n_phy;
	u32		srs_sz;
	u32		slot_width;
};

struct mvs_err_info {
	__le32			flags;
	__le32			flags2;
};

struct mvs_prd {
	__le64			addr;		/* 64-bit buffer address */
	__le32			reserved;
	__le32			len;		/* 16-bit length */
};

struct mvs_cmd_hdr {
	__le32			flags;		/* PRD tbl len; SAS, SATA ctl */
	__le32			lens;		/* cmd, max resp frame len */
	__le32			tags;		/* targ port xfer tag; tag */
	__le32			data_len;	/* data xfer len */
	__le64			cmd_tbl;	/* command table address */
	__le64			open_frame;	/* open addr frame address */
	__le64			status_buf;	/* status buffer address */
	__le64			prd_tbl;	/* PRD tbl address */
	__le32			reserved[4];
};

struct mvs_port {
	struct asd_sas_port	sas_port;
	u8			port_attached;
	u8			taskfileset;
	u8			wide_port_phymap;
	struct list_head	list;
};

struct mvs_phy {
	struct mvs_port		*port;
	struct asd_sas_phy	sas_phy;
	struct sas_identify	identify;
	struct scsi_device	*sdev;
	u64		dev_sas_addr;
	u64		att_dev_sas_addr;
	u32		att_dev_info;
	u32		dev_info;
	u32		phy_type;
	u32		phy_status;
	u32		irq_status;
	u32		frame_rcvd_size;
	u8		frame_rcvd[32];
	u8		phy_attached;
	enum sas_linkrate	minimum_linkrate;
	enum sas_linkrate	maximum_linkrate;
};

struct mvs_slot_info {
	struct list_head	list;
	struct sas_task		*task;
	u32			n_elem;
	u32			tx;

	/* DMA buffer for storing cmd tbl, open addr frame, status buffer,
	 * and PRD table
	 */
	void			*buf;
	dma_addr_t		buf_dma;
#if _MV_DUMP
	u32			cmd_size;
#endif

	void			*response;
	struct mvs_port		*port;
};

struct mvs_info {
	unsigned long		flags;

	spinlock_t		lock;		/* host-wide lock */
	struct pci_dev		*pdev;		/* our device */
	void __iomem		*regs;		/* enhanced mode registers */
	void __iomem		*peri_regs;	/* peripheral registers */

	u8			sas_addr[SAS_ADDR_SIZE];
	struct sas_ha_struct	sas;		/* SCSI/SAS glue */
	struct Scsi_Host	*shost;

	__le32			*tx;		/* TX (delivery) DMA ring */
	dma_addr_t		tx_dma;
	u32			tx_prod;	/* cached next-producer idx */

	__le32			*rx;		/* RX (completion) DMA ring */
	dma_addr_t		rx_dma;
	u32			rx_cons;	/* RX consumer idx */

	__le32			*rx_fis;	/* RX'd FIS area */
	dma_addr_t		rx_fis_dma;

	struct mvs_cmd_hdr	*slot;	/* DMA command header slots */
	dma_addr_t		slot_dma;

	const struct mvs_chip_info *chip;

	u8			tags[MVS_SLOTS];
	struct mvs_slot_info	slot_info[MVS_SLOTS];
				/* further per-slot information */
	struct mvs_phy		phy[MVS_MAX_PHYS];
	struct mvs_port		port[MVS_MAX_PHYS];
#ifdef MVS_USE_TASKLET
	struct tasklet_struct	tasklet;
#endif
};

static int mvs_phy_control(struct asd_sas_phy *sas_phy, enum phy_func func,
			   void *funcdata);
static u32 mvs_read_phy_ctl(struct mvs_info *mvi, u32 port);
static void mvs_write_phy_ctl(struct mvs_info *mvi, u32 port, u32 val);
static u32 mvs_read_port_irq_stat(struct mvs_info *mvi, u32 port);
static void mvs_write_port_irq_stat(struct mvs_info *mvi, u32 port, u32 val);
static void mvs_write_port_irq_mask(struct mvs_info *mvi, u32 port, u32 val);
static u32 mvs_read_port_irq_mask(struct mvs_info *mvi, u32 port);

static u32 mvs_is_phy_ready(struct mvs_info *mvi, int i);
static void mvs_detect_porttype(struct mvs_info *mvi, int i);
static void mvs_update_phyinfo(struct mvs_info *mvi, int i, int get_st);
static void mvs_release_task(struct mvs_info *mvi, int phy_no);

static int mvs_scan_finished(struct Scsi_Host *, unsigned long);
static void mvs_scan_start(struct Scsi_Host *);
static int mvs_slave_configure(struct scsi_device *sdev);

static struct scsi_transport_template *mvs_stt;

static const struct mvs_chip_info mvs_chips[] = {
	[chip_6320] =		{ 2, 16, 9  },
	[chip_6440] =		{ 4, 16, 9  },
	[chip_6480] =		{ 8, 32, 10 },
};

static struct scsi_host_template mvs_sht = {
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.queuecommand		= sas_queuecommand,
	.target_alloc		= sas_target_alloc,
	.slave_configure	= mvs_slave_configure,
	.slave_destroy		= sas_slave_destroy,
	.scan_finished		= mvs_scan_finished,
	.scan_start		= mvs_scan_start,
	.change_queue_depth	= sas_change_queue_depth,
	.change_queue_type	= sas_change_queue_type,
	.bios_param		= sas_bios_param,
	.can_queue		= 1,
	.cmd_per_lun		= 1,
	.this_id		= -1,
	.sg_tablesize		= SG_ALL,
	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
	.use_clustering		= ENABLE_CLUSTERING,
	.eh_device_reset_handler	= sas_eh_device_reset_handler,
	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
	.slave_alloc		= sas_slave_alloc,
	.target_destroy		= sas_target_destroy,
	.ioctl			= sas_ioctl,
};

static void mvs_hexdump(u32 size, u8 *data, u32 baseaddr)
{
	u32 i;
	u32 run;
	u32 offset;

	offset = 0;
	while (size) {
		printk("%08X : ", baseaddr + offset);
		if (size >= 16)
			run = 16;
		else
			run = size;
		size -= run;
		for (i = 0; i < 16; i++) {
			if (i < run)
				printk("%02X ", (u32)data[i]);
			else
				printk("   ");
		}
		printk(": ");
		for (i = 0; i < run; i++)
			printk("%c", isalnum(data[i]) ? data[i] : '.');
		printk("\n");
		data = &data[16];
		offset += run;
	}
	printk("\n");
}

#if _MV_DUMP
static void mvs_hba_sb_dump(struct mvs_info *mvi, u32 tag,
				   enum sas_protocol proto)
{
	u32 offset;
	struct pci_dev *pdev = mvi->pdev;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];

	offset = slot->cmd_size + MVS_OAF_SZ +
	    sizeof(struct mvs_prd) * slot->n_elem;
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Status buffer[%d] :\n",
			tag);
	mvs_hexdump(32, (u8 *) slot->response,
		    (u32) slot->buf_dma + offset);
}
#endif

static void mvs_hba_memory_dump(struct mvs_info *mvi, u32 tag,
				enum sas_protocol proto)
{
#if _MV_DUMP
	u32 sz, w_ptr;
	u64 addr;
	void __iomem *regs = mvi->regs;
	struct pci_dev *pdev = mvi->pdev;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];

	/*Delivery Queue */
	sz = mr32(TX_CFG) & TX_RING_SZ_MASK;
	w_ptr = slot->tx;
	addr = mr32(TX_HI) << 16 << 16 | mr32(TX_LO);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Delivery Queue Size=%04d , WRT_PTR=%04X\n", sz, w_ptr);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Delivery Queue Base Address=0x%llX (PA)"
		"(tx_dma=0x%llX), Entry=%04d\n",
		addr, mvi->tx_dma, w_ptr);
	mvs_hexdump(sizeof(u32), (u8 *)(&mvi->tx[mvi->tx_prod]),
			(u32) mvi->tx_dma + sizeof(u32) * w_ptr);
	/*Command List */
	addr = mvi->slot_dma;
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Command List Base Address=0x%llX (PA)"
		"(slot_dma=0x%llX), Header=%03d\n",
		addr, slot->buf_dma, tag);
	dev_printk(KERN_DEBUG, &pdev->dev, "Command Header[%03d]:\n", tag);
	/*mvs_cmd_hdr */
	mvs_hexdump(sizeof(struct mvs_cmd_hdr), (u8 *)(&mvi->slot[tag]),
		(u32) mvi->slot_dma + tag * sizeof(struct mvs_cmd_hdr));
	/*1.command table area */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Command Table :\n");
	mvs_hexdump(slot->cmd_size, (u8 *) slot->buf, (u32) slot->buf_dma);
	/*2.open address frame area */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Open Address Frame :\n");
	mvs_hexdump(MVS_OAF_SZ, (u8 *) slot->buf + slot->cmd_size,
				(u32) slot->buf_dma + slot->cmd_size);
	/*3.status buffer */
	mvs_hba_sb_dump(mvi, tag, proto);
	/*4.PRD table */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->PRD table :\n");
	mvs_hexdump(sizeof(struct mvs_prd) * slot->n_elem,
		(u8 *) slot->buf + slot->cmd_size + MVS_OAF_SZ,
		(u32) slot->buf_dma + slot->cmd_size + MVS_OAF_SZ);
#endif
}

static void mvs_hba_cq_dump(struct mvs_info *mvi)
{
#if (_MV_DUMP > 2)
	u64 addr;
	void __iomem *regs = mvi->regs;
	struct pci_dev *pdev = mvi->pdev;
	u32 entry = mvi->rx_cons + 1;
	u32 rx_desc = le32_to_cpu(mvi->rx[entry]);

	/*Completion Queue */
	addr = mr32(RX_HI) << 16 << 16 | mr32(RX_LO);
	dev_printk(KERN_DEBUG, &pdev->dev, "Completion Task = 0x%p\n",
		   mvi->slot_info[rx_desc & RXQ_SLOT_MASK].task);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Completion List Base Address=0x%llX (PA), "
		"CQ_Entry=%04d, CQ_WP=0x%08X\n",
		addr, entry - 1, mvi->rx[0]);
	mvs_hexdump(sizeof(u32), (u8 *)(&rx_desc),
		    mvi->rx_dma + sizeof(u32) * entry);
#endif
}

static void mvs_hba_interrupt_enable(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp;

	tmp = mr32(GBL_CTL);

	mw32(GBL_CTL, tmp | INT_EN);
}

static void mvs_hba_interrupt_disable(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp;

	tmp = mr32(GBL_CTL);

	mw32(GBL_CTL, tmp & ~INT_EN);
}

static int mvs_int_rx(struct mvs_info *mvi, bool self_clear);

/* move to PCI layer or libata core? */
static int pci_go_64(struct pci_dev *pdev)
{
	int rc;

	if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
		if (rc) {
			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
			if (rc) {
				dev_printk(KERN_ERR, &pdev->dev,
					   "64-bit DMA enable failed\n");
				return rc;
			}
		}
	} else {
		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit DMA enable failed\n");
			return rc;
		}
		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit consistent DMA enable failed\n");
			return rc;
		}
	}

	return rc;
}

static int mvs_find_tag(struct mvs_info *mvi, struct sas_task *task, u32 *tag)
{
	if (task->lldd_task) {
		struct mvs_slot_info *slot;
		slot = (struct mvs_slot_info *) task->lldd_task;
		*tag = slot - mvi->slot_info;
		return 1;
	}
	return 0;
}

static void mvs_tag_clear(struct mvs_info *mvi, u32 tag)
{
	void *bitmap = (void *) &mvi->tags;
	clear_bit(tag, bitmap);
}

static void mvs_tag_free(struct mvs_info *mvi, u32 tag)
{
	mvs_tag_clear(mvi, tag);
}

static void mvs_tag_set(struct mvs_info *mvi, unsigned int tag)
{
	void *bitmap = (void *) &mvi->tags;
	set_bit(tag, bitmap);
}

static int mvs_tag_alloc(struct mvs_info *mvi, u32 *tag_out)
{
	unsigned int index, tag;
	void *bitmap = (void *) &mvi->tags;

	index = find_first_zero_bit(bitmap, MVS_SLOTS);
	tag = index;
	if (tag >= MVS_SLOTS)
		return -SAS_QUEUE_FULL;
	mvs_tag_set(mvi, tag);
	*tag_out = tag;
	return 0;
}

static void mvs_tag_init(struct mvs_info *mvi)
{
	int i;
	for (i = 0; i < MVS_SLOTS; ++i)
		mvs_tag_clear(mvi, i);
}

#ifndef MVS_DISABLE_NVRAM
static int mvs_eep_read(void __iomem *regs, u32 addr, u32 *data)
{
	int timeout = 1000;

	if (addr & ~SPI_ADDR_MASK)
		return -EINVAL;

	writel(addr, regs + SPI_CMD);
	writel(TWSI_RD, regs + SPI_CTL);

	while (timeout-- > 0) {
		if (readl(regs + SPI_CTL) & TWSI_RDY) {
			*data = readl(regs + SPI_DATA);
			return 0;
		}

		udelay(10);
	}

	return -EBUSY;
}

static int mvs_eep_read_buf(void __iomem *regs, u32 addr,
			    void *buf, u32 buflen)
{
	u32 addr_end, tmp_addr, i, j;
	u32 tmp = 0;
	int rc;
	u8 *tmp8, *buf8 = buf;

	addr_end = addr + buflen;
	tmp_addr = ALIGN(addr, 4);
	if (addr > 0xff)
		return -EINVAL;

	j = addr & 0x3;
	if (j) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		tmp8 = (u8 *)&tmp;
		for (i = j; i < 4; i++)
			*buf8++ = tmp8[i];

		tmp_addr += 4;
	}

	for (j = ALIGN(addr_end, 4); tmp_addr < j; tmp_addr += 4) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		memcpy(buf8, &tmp, 4);
		buf8 += 4;
	}

	if (tmp_addr < addr_end) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		tmp8 = (u8 *)&tmp;
		j = addr_end - tmp_addr;
		for (i = 0; i < j; i++)
			*buf8++ = tmp8[i];

		tmp_addr += 4;
	}

	return 0;
}
#endif

static int mvs_nvram_read(struct mvs_info *mvi, u32 addr,
			  void *buf, u32 buflen)
{
#ifndef MVS_DISABLE_NVRAM
	void __iomem *regs = mvi->regs;
	int rc, i;
	u32 sum;
	u8 hdr[2], *tmp;
	const char *msg;

	rc = mvs_eep_read_buf(regs, addr, &hdr, 2);
	if (rc) {
		msg = "nvram hdr read failed";
		goto err_out;
	}
	rc = mvs_eep_read_buf(regs, addr + 2, buf, buflen);
	if (rc) {
		msg = "nvram read failed";
		goto err_out;
	}

	if (hdr[0] != 0x5A) {
		/* entry id */
		msg = "invalid nvram entry id";
		rc = -ENOENT;
		goto err_out;
	}

	tmp = buf;
	sum = ((u32)hdr[0]) + ((u32)hdr[1]);
	for (i = 0; i < buflen; i++)
		sum += ((u32)tmp[i]);

	if (sum) {
		msg = "nvram checksum failure";
		rc = -EILSEQ;
		goto err_out;
	}

	return 0;

err_out:
	dev_printk(KERN_ERR, &mvi->pdev->dev, "%s", msg);
	return rc;
#else
	/* FIXME , For SAS target mode */
	memcpy(buf, "\x50\x05\x04\x30\x11\xab\x00\x00", 8);
	return 0;
#endif
}

static void mvs_bytes_dmaed(struct mvs_info *mvi, int i)
{
	struct mvs_phy *phy = &mvi->phy[i];
	struct asd_sas_phy *sas_phy = mvi->sas.sas_phy[i];

	if (!phy->phy_attached)
		return;

	if (sas_phy->phy) {
		struct sas_phy *sphy = sas_phy->phy;

		sphy->negotiated_linkrate = sas_phy->linkrate;
		sphy->minimum_linkrate = phy->minimum_linkrate;
		sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS;
		sphy->maximum_linkrate = phy->maximum_linkrate;
		sphy->maximum_linkrate_hw = SAS_LINK_RATE_3_0_GBPS;
	}

	if (phy->phy_type & PORT_TYPE_SAS) {
		struct sas_identify_frame *id;

		id = (struct sas_identify_frame *)phy->frame_rcvd;
		id->dev_type = phy->identify.device_type;
		id->initiator_bits = SAS_PROTOCOL_ALL;
		id->target_bits = phy->identify.target_port_protocols;
	} else if (phy->phy_type & PORT_TYPE_SATA) {
		/* TODO */
	}
	mvi->sas.sas_phy[i]->frame_rcvd_size = phy->frame_rcvd_size;
	mvi->sas.notify_port_event(mvi->sas.sas_phy[i],
				   PORTE_BYTES_DMAED);
}

static int mvs_scan_finished(struct Scsi_Host *shost, unsigned long time)
{
	/* give the phy enabling interrupt event time to come in (1s
	 * is empirically about all it takes) */
	if (time < HZ)
		return 0;
	/* Wait for discovery to finish */
	scsi_flush_work(shost);
	return 1;
}

static void mvs_scan_start(struct Scsi_Host *shost)
{
	int i;
	struct mvs_info *mvi = SHOST_TO_SAS_HA(shost)->lldd_ha;

	for (i = 0; i < mvi->chip->n_phy; ++i) {
		mvs_bytes_dmaed(mvi, i);
	}
}

static int mvs_slave_configure(struct scsi_device *sdev)
{
	struct domain_device *dev = sdev_to_domain_dev(sdev);
	int ret = sas_slave_configure(sdev);

	if (ret)
		return ret;

	if (dev_is_sata(dev)) {
		/* struct ata_port *ap = dev->sata_dev.ap; */
		/* struct ata_device *adev = ap->link.device; */

		/* clamp at no NCQ for the time being */
		/* adev->flags |= ATA_DFLAG_NCQ_OFF; */
		scsi_adjust_queue_depth(sdev, MSG_SIMPLE_TAG, 1);
	}
	return 0;
}

static void mvs_int_port(struct mvs_info *mvi, int phy_no, u32 events)
{
	struct pci_dev *pdev = mvi->pdev;
	struct sas_ha_struct *sas_ha = &mvi->sas;
	struct mvs_phy *phy = &mvi->phy[phy_no];
	struct asd_sas_phy *sas_phy = &phy->sas_phy;

	phy->irq_status = mvs_read_port_irq_stat(mvi, phy_no);
	/*
	* events is port event now ,
	* we need check the interrupt status which belongs to per port.
	*/
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Port %d Event = %X\n",
		phy_no, phy->irq_status);

	if (phy->irq_status & (PHYEV_POOF | PHYEV_DEC_ERR)) {
		mvs_release_task(mvi, phy_no);
		if (!mvs_is_phy_ready(mvi, phy_no)) {
			sas_phy_disconnected(sas_phy);
			sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL);
			dev_printk(KERN_INFO, &pdev->dev,
				"Port %d Unplug Notice\n", phy_no);

		} else
			mvs_phy_control(sas_phy, PHY_FUNC_LINK_RESET, NULL);
	}
	if (!(phy->irq_status & PHYEV_DEC_ERR)) {
		if (phy->irq_status & PHYEV_COMWAKE) {
			u32 tmp = mvs_read_port_irq_mask(mvi, phy_no);
			mvs_write_port_irq_mask(mvi, phy_no,
						tmp | PHYEV_SIG_FIS);
		}
		if (phy->irq_status & (PHYEV_SIG_FIS | PHYEV_ID_DONE)) {
			phy->phy_status = mvs_is_phy_ready(mvi, phy_no);
			if (phy->phy_status) {
				mvs_detect_porttype(mvi, phy_no);

				if (phy->phy_type & PORT_TYPE_SATA) {
					u32 tmp = mvs_read_port_irq_mask(mvi,
								phy_no);
					tmp &= ~PHYEV_SIG_FIS;
					mvs_write_port_irq_mask(mvi,
								phy_no, tmp);
				}

				mvs_update_phyinfo(mvi, phy_no, 0);
				sas_ha->notify_phy_event(sas_phy,
							PHYE_OOB_DONE);
				mvs_bytes_dmaed(mvi, phy_no);
			} else {
				dev_printk(KERN_DEBUG, &pdev->dev,
					"plugin interrupt but phy is gone\n");
				mvs_phy_control(sas_phy, PHY_FUNC_LINK_RESET,
							NULL);
			}
		} else if (phy->irq_status & PHYEV_BROAD_CH) {
			mvs_release_task(mvi, phy_no);
			sas_ha->notify_port_event(sas_phy,
						PORTE_BROADCAST_RCVD);
		}
	}
	mvs_write_port_irq_stat(mvi, phy_no, phy->irq_status);
}

static void mvs_int_sata(struct mvs_info *mvi)
{
	/* FIXME */
}

static void mvs_slot_free(struct mvs_info *mvi, struct sas_task *task,
			  struct mvs_slot_info *slot, u32 slot_idx)
{
	if (!sas_protocol_ata(task->task_proto))
		if (slot->n_elem)
			pci_unmap_sg(mvi->pdev, task->scatter,
				     slot->n_elem, task->data_dir);

	switch (task->task_proto) {
	case SAS_PROTOCOL_SMP:
		pci_unmap_sg(mvi->pdev, &task->smp_task.smp_resp, 1,
			     PCI_DMA_FROMDEVICE);
		pci_unmap_sg(mvi->pdev, &task->smp_task.smp_req, 1,
			     PCI_DMA_TODEVICE);
		break;

	case SAS_PROTOCOL_SATA:
	case SAS_PROTOCOL_STP:
	case SAS_PROTOCOL_SSP:
	default:
		/* do nothing */
		break;
	}

	slot->task = NULL;
	mvs_tag_clear(mvi, slot_idx);
}

static void mvs_slot_err(struct mvs_info *mvi, struct sas_task *task,
			 u32 slot_idx)
{
	struct mvs_slot_info *slot = &mvi->slot_info[slot_idx];
	u64 err_dw0 = *(u32 *) slot->response;
	void __iomem *regs = mvi->regs;
	u32 tmp;

	if (err_dw0 & CMD_ISS_STPD)
		if (sas_protocol_ata(task->task_proto)) {
			tmp = mr32(INT_STAT_SRS);
			mw32(INT_STAT_SRS, tmp & 0xFFFF);
		}

	mvs_hba_sb_dump(mvi, slot_idx, task->task_proto);
}

static int mvs_slot_complete(struct mvs_info *mvi, u32 rx_desc)
{
	u32 slot_idx = rx_desc & RXQ_SLOT_MASK;
	struct mvs_slot_info *slot = &mvi->slot_info[slot_idx];
	struct sas_task *task = slot->task;
	struct task_status_struct *tstat = &task->task_status;
	struct mvs_port *port = &mvi->port[task->dev->port->id];
	bool aborted;
	void *to;

	spin_lock(&task->task_state_lock);
	aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
	if (!aborted) {
		task->task_state_flags &=
		    ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
		task->task_state_flags |= SAS_TASK_STATE_DONE;
	}
	spin_unlock(&task->task_state_lock);

	if (aborted)
		return -1;

	memset(tstat, 0, sizeof(*tstat));
	tstat->resp = SAS_TASK_COMPLETE;


	if (unlikely(!port->port_attached)) {
		tstat->stat = SAS_PHY_DOWN;
		goto out;
	}

	/* error info record present */
	if ((rx_desc & RXQ_ERR) && (*(u64 *) slot->response)) {
		tstat->stat = SAM_CHECK_COND;
		mvs_slot_err(mvi, task, slot_idx);
		goto out;
	}

	switch (task->task_proto) {
	case SAS_PROTOCOL_SSP:
		/* hw says status == 0, datapres == 0 */
		if (rx_desc & RXQ_GOOD) {
			tstat->stat = SAM_GOOD;
			tstat->resp = SAS_TASK_COMPLETE;
		}
		/* response frame present */
		else if (rx_desc & RXQ_RSP) {
			struct ssp_response_iu *iu =
			    slot->response + sizeof(struct mvs_err_info);
			sas_ssp_task_response(&mvi->pdev->dev, task, iu);
		}

		/* should never happen? */
		else
			tstat->stat = SAM_CHECK_COND;
		break;

	case SAS_PROTOCOL_SMP: {
			struct scatterlist *sg_resp = &task->smp_task.smp_resp;
			tstat->stat = SAM_GOOD;
			to = kmap_atomic(sg_page(sg_resp), KM_IRQ0);
			memcpy(to + sg_resp->offset,
				slot->response + sizeof(struct mvs_err_info),
				sg_dma_len(sg_resp));
			kunmap_atomic(to, KM_IRQ0);
			break;
		}

	case SAS_PROTOCOL_SATA:
	case SAS_PROTOCOL_STP:
	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: {
			struct ata_task_resp *resp =
			    (struct ata_task_resp *)tstat->buf;

			if ((rx_desc & (RXQ_DONE | RXQ_ERR | RXQ_ATTN)) ==
			    RXQ_DONE)
				tstat->stat = SAM_GOOD;
			else
				tstat->stat = SAM_CHECK_COND;

			resp->frame_len = sizeof(struct dev_to_host_fis);
			memcpy(&resp->ending_fis[0],
			       SATA_RECEIVED_D2H_FIS(port->taskfileset),
			       sizeof(struct dev_to_host_fis));
			if (resp->ending_fis[2] & ATA_ERR)
				mvs_hexdump(16, resp->ending_fis, 0);
			break;
		}

	default:
		tstat->stat = SAM_CHECK_COND;
		break;
	}

out:
	mvs_slot_free(mvi, task, slot, slot_idx);
	task->task_done(task);
	return tstat->stat;
}

static void mvs_int_full(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp, stat;
	int i;

	stat = mr32(INT_STAT);

	mvs_int_rx(mvi, false);

	for (i = 0; i < MVS_MAX_PORTS; i++) {
		tmp = (stat >> i) & (CINT_PORT | CINT_PORT_STOPPED);
		if (tmp)
			mvs_int_port(mvi, i, tmp);
	}

	if (stat & CINT_SRS)
		mvs_int_sata(mvi);

	mw32(INT_STAT, stat);
}

static int mvs_int_rx(struct mvs_info *mvi, bool self_clear)
{
	void __iomem *regs = mvi->regs;
	u32 rx_prod_idx, rx_desc;
	bool attn = false;
	struct pci_dev *pdev = mvi->pdev;

	/* the first dword in the RX ring is special: it contains
	 * a mirror of the hardware's RX producer index, so that
	 * we don't have to stall the CPU reading that register.
	 * The actual RX ring is offset by one dword, due to this.
	 */
	rx_prod_idx = mr32(RX_CONS_IDX) & RX_RING_SZ_MASK;
	if (rx_prod_idx == 0xfff) {	/* h/w hasn't touched RX ring yet */
		mvi->rx_cons = 0xfff;
		return 0;
	}

	/* The CMPL_Q may come late, read from register and try again
	* note: if coalescing is enabled,
	* it will need to read from register every time for sure
	*/
	if (mvi->rx_cons == rx_prod_idx)
		return 0;

	if (mvi->rx_cons == 0xfff)
		mvi->rx_cons = MVS_RX_RING_SZ - 1;

	while (mvi->rx_cons != rx_prod_idx) {

		/* increment our internal RX consumer pointer */
		mvi->rx_cons = (mvi->rx_cons + 1) & (MVS_RX_RING_SZ - 1);

		rx_desc = le32_to_cpu(mvi->rx[mvi->rx_cons + 1]);

		mvs_hba_cq_dump(mvi);

		if (likely(rx_desc & RXQ_DONE))
			mvs_slot_complete(mvi, rx_desc);
		if (rx_desc & RXQ_ATTN) {
			attn = true;
			dev_printk(KERN_DEBUG, &pdev->dev, "ATTN %X\n",
				rx_desc);
		} else if (rx_desc & RXQ_ERR) {
			dev_printk(KERN_DEBUG, &pdev->dev, "RXQ_ERR %X\n",
				rx_desc);
		}
	}

	if (attn && self_clear)
		mvs_int_full(mvi);

	return 0;
}

static irqreturn_t mvs_interrupt(int irq, void *opaque)
{
	struct mvs_info *mvi = opaque;
	void __iomem *regs = mvi->regs;
	u32 stat;

	stat = mr32(GBL_INT_STAT);

	/* clear CMD_CMPLT ASAP */
	mw32_f(INT_STAT, CINT_DONE);

	if (stat == 0 || stat == 0xffffffff)
		return IRQ_NONE;

	spin_lock(&mvi->lock);

	mvs_int_full(mvi);

	spin_unlock(&mvi->lock);

	return IRQ_HANDLED;
}

#ifndef MVS_DISABLE_MSI
static irqreturn_t mvs_msi_interrupt(int irq, void *opaque)
{
	struct mvs_info *mvi = opaque;

	spin_lock(&mvi->lock);

	mvs_int_rx(mvi, true);

	spin_unlock(&mvi->lock);

	return IRQ_HANDLED;
}
#endif

struct mvs_task_exec_info {
	struct sas_task *task;
	struct mvs_cmd_hdr *hdr;
	struct mvs_port *port;
	u32 tag;
	int n_elem;
};

static int mvs_task_prep_smp(struct mvs_info *mvi,
			     struct mvs_task_exec_info *tei)
{
	int elem, rc, i;
	struct sas_task *task = tei->task;
	struct mvs_cmd_hdr *hdr = tei->hdr;
	struct scatterlist *sg_req, *sg_resp;
	u32 req_len, resp_len, tag = tei->tag;
	void *buf_tmp;
	u8 *buf_oaf;
	dma_addr_t buf_tmp_dma;
	struct mvs_prd *buf_prd;
	struct scatterlist *sg;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];
	struct asd_sas_port *sas_port = task->dev->port;
	u32 flags = (tei->n_elem << MCH_PRD_LEN_SHIFT);<commit_msg>[SCSI] mvsas : interrupt handling<commit_after>/*
	mvsas.c - Marvell 88SE6440 SAS/SATA support

	Copyright 2007 Red Hat, Inc.
	Copyright 2008 Marvell. <kewei<EMAIL>>

	This program is free software; you can redistribute it and/or
	modify it under the terms of the GNU General Public License as
	published by the Free Software Foundation; either version 2,
	or (at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty
	of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	See the GNU General Public License for more details.

	You should have received a copy of the GNU General Public
	License along with this program; see the file COPYING.	If not,
	write to the Free Software Foundation, 675 Mass Ave, Cambridge,
	MA 02139, USA.

	---------------------------------------------------------------

	Random notes:
	* hardware supports controlling the endian-ness of data
	  structures.  this permits elimination of all the le32_to_cpu()
	  and cpu_to_le32() conversions.

 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/ctype.h>
#include <scsi/libsas.h>
#include <scsi/scsi_tcq.h>
#include <scsi/sas_ata.h>
#include <asm/io.h>

#define DRV_NAME	"mvsas"
#define DRV_VERSION	"0.5.2"
#define _MV_DUMP	0
#define MVS_DISABLE_NVRAM
#define MVS_DISABLE_MSI

#define mr32(reg)	readl(regs + MVS_##reg)
#define mw32(reg,val)	writel((val), regs + MVS_##reg)
#define mw32_f(reg,val)	do {			\
	writel((val), regs + MVS_##reg);	\
	readl(regs + MVS_##reg);		\
	} while (0)

#define MVS_ID_NOT_MAPPED	0x7f
#define MVS_CHIP_SLOT_SZ	(1U << mvi->chip->slot_width)

/* offset for D2H FIS in the Received FIS List Structure */
#define SATA_RECEIVED_D2H_FIS(reg_set)	\
	((void *) mvi->rx_fis + 0x400 + 0x100 * reg_set + 0x40)
#define SATA_RECEIVED_PIO_FIS(reg_set)	\
	((void *) mvi->rx_fis + 0x400 + 0x100 * reg_set + 0x20)
#define UNASSOC_D2H_FIS(id)		\
	((void *) mvi->rx_fis + 0x100 * id)

#define for_each_phy(__lseq_mask, __mc, __lseq, __rest)			\
	for ((__mc) = (__lseq_mask), (__lseq) = 0;			\
					(__mc) != 0 && __rest;		\
					(++__lseq), (__mc) >>= 1)

/* driver compile-time configuration */
enum driver_configuration {
	MVS_TX_RING_SZ		= 1024,	/* TX ring size (12-bit) */
	MVS_RX_RING_SZ		= 1024, /* RX ring size (12-bit) */
					/* software requires power-of-2
					   ring size */

	MVS_SLOTS		= 512,	/* command slots */
	MVS_SLOT_BUF_SZ		= 8192, /* cmd tbl + IU + status + PRD */
	MVS_SSP_CMD_SZ		= 64,	/* SSP command table buffer size */
	MVS_ATA_CMD_SZ		= 96,	/* SATA command table buffer size */
	MVS_OAF_SZ		= 64,	/* Open address frame buffer size */

	MVS_RX_FIS_COUNT	= 17,	/* Optional rx'd FISs (max 17) */

	MVS_QUEUE_SIZE		= 30,	/* Support Queue depth */
	MVS_CAN_QUEUE		= MVS_SLOTS - 1,	/* SCSI Queue depth */
};

/* unchangeable hardware details */
enum hardware_details {
	MVS_MAX_PHYS		= 8,	/* max. possible phys */
	MVS_MAX_PORTS		= 8,	/* max. possible ports */
	MVS_RX_FISL_SZ		= 0x400 + (MVS_RX_FIS_COUNT * 0x100),
};

/* peripheral registers (BAR2) */
enum peripheral_registers {
	SPI_CTL			= 0x10,	/* EEPROM control */
	SPI_CMD			= 0x14,	/* EEPROM command */
	SPI_DATA		= 0x18, /* EEPROM data */
};

enum peripheral_register_bits {
	TWSI_RDY		= (1U << 7),	/* EEPROM interface ready */
	TWSI_RD			= (1U << 4),	/* EEPROM read access */

	SPI_ADDR_MASK		= 0x3ffff,	/* bits 17:0 */
};

/* enhanced mode registers (BAR4) */
enum hw_registers {
	MVS_GBL_CTL		= 0x04,  /* global control */
	MVS_GBL_INT_STAT	= 0x08,  /* global irq status */
	MVS_GBL_PI		= 0x0C,  /* ports implemented bitmask */
	MVS_GBL_PORT_TYPE	= 0xa0,  /* port type */

	MVS_CTL			= 0x100, /* SAS/SATA port configuration */
	MVS_PCS			= 0x104, /* SAS/SATA port control/status */
	MVS_CMD_LIST_LO		= 0x108, /* cmd list addr */
	MVS_CMD_LIST_HI		= 0x10C,
	MVS_RX_FIS_LO		= 0x110, /* RX FIS list addr */
	MVS_RX_FIS_HI		= 0x114,

	MVS_TX_CFG		= 0x120, /* TX configuration */
	MVS_TX_LO		= 0x124, /* TX (delivery) ring addr */
	MVS_TX_HI		= 0x128,

	MVS_TX_PROD_IDX		= 0x12C, /* TX producer pointer */
	MVS_TX_CONS_IDX		= 0x130, /* TX consumer pointer (RO) */
	MVS_RX_CFG		= 0x134, /* RX configuration */
	MVS_RX_LO		= 0x138, /* RX (completion) ring addr */
	MVS_RX_HI		= 0x13C,
	MVS_RX_CONS_IDX		= 0x140, /* RX consumer pointer (RO) */

	MVS_INT_COAL		= 0x148, /* Int coalescing config */
	MVS_INT_COAL_TMOUT	= 0x14C, /* Int coalescing timeout */
	MVS_INT_STAT		= 0x150, /* Central int status */
	MVS_INT_MASK		= 0x154, /* Central int enable */
	MVS_INT_STAT_SRS	= 0x158, /* SATA register set status */
	MVS_INT_MASK_SRS	= 0x15C,

					 /* ports 1-3 follow after this */
	MVS_P0_INT_STAT		= 0x160, /* port0 interrupt status */
	MVS_P0_INT_MASK		= 0x164, /* port0 interrupt mask */
	MVS_P4_INT_STAT		= 0x200, /* Port 4 interrupt status */
	MVS_P4_INT_MASK		= 0x204, /* Port 4 interrupt enable mask */

					 /* ports 1-3 follow after this */
	MVS_P0_SER_CTLSTAT	= 0x180, /* port0 serial control/status */
	MVS_P4_SER_CTLSTAT	= 0x220, /* port4 serial control/status */

	MVS_CMD_ADDR		= 0x1B8, /* Command register port (addr) */
	MVS_CMD_DATA		= 0x1BC, /* Command register port (data) */

					 /* ports 1-3 follow after this */
	MVS_P0_CFG_ADDR		= 0x1C0, /* port0 phy register address */
	MVS_P0_CFG_DATA		= 0x1C4, /* port0 phy register data */
	MVS_P4_CFG_ADDR		= 0x230, /* Port 4 config address */
	MVS_P4_CFG_DATA		= 0x234, /* Port 4 config data */

					 /* ports 1-3 follow after this */
	MVS_P0_VSR_ADDR		= 0x1E0, /* port0 VSR address */
	MVS_P0_VSR_DATA		= 0x1E4, /* port0 VSR data */
	MVS_P4_VSR_ADDR		= 0x250, /* port 4 VSR addr */
	MVS_P4_VSR_DATA		= 0x254, /* port 4 VSR data */
};

enum hw_register_bits {
	/* MVS_GBL_CTL */
	INT_EN			= (1U << 1),	/* Global int enable */
	HBA_RST			= (1U << 0),	/* HBA reset */

	/* MVS_GBL_INT_STAT */
	INT_XOR			= (1U << 4),	/* XOR engine event */
	INT_SAS_SATA		= (1U << 0),	/* SAS/SATA event */

	/* MVS_GBL_PORT_TYPE */			/* shl for ports 1-3 */
	SATA_TARGET		= (1U << 16),	/* port0 SATA target enable */
	MODE_AUTO_DET_PORT7 = (1U << 15),	/* port0 SAS/SATA autodetect */
	MODE_AUTO_DET_PORT6 = (1U << 14),
	MODE_AUTO_DET_PORT5 = (1U << 13),
	MODE_AUTO_DET_PORT4 = (1U << 12),
	MODE_AUTO_DET_PORT3 = (1U << 11),
	MODE_AUTO_DET_PORT2 = (1U << 10),
	MODE_AUTO_DET_PORT1 = (1U << 9),
	MODE_AUTO_DET_PORT0 = (1U << 8),
	MODE_AUTO_DET_EN    =	MODE_AUTO_DET_PORT0 | MODE_AUTO_DET_PORT1 |
				MODE_AUTO_DET_PORT2 | MODE_AUTO_DET_PORT3 |
				MODE_AUTO_DET_PORT4 | MODE_AUTO_DET_PORT5 |
				MODE_AUTO_DET_PORT6 | MODE_AUTO_DET_PORT7,
	MODE_SAS_PORT7_MASK = (1U << 7),  /* port0 SAS(1), SATA(0) mode */
	MODE_SAS_PORT6_MASK = (1U << 6),
	MODE_SAS_PORT5_MASK = (1U << 5),
	MODE_SAS_PORT4_MASK = (1U << 4),
	MODE_SAS_PORT3_MASK = (1U << 3),
	MODE_SAS_PORT2_MASK = (1U << 2),
	MODE_SAS_PORT1_MASK = (1U << 1),
	MODE_SAS_PORT0_MASK = (1U << 0),
	MODE_SAS_SATA	=	MODE_SAS_PORT0_MASK | MODE_SAS_PORT1_MASK |
				MODE_SAS_PORT2_MASK | MODE_SAS_PORT3_MASK |
				MODE_SAS_PORT4_MASK | MODE_SAS_PORT5_MASK |
				MODE_SAS_PORT6_MASK | MODE_SAS_PORT7_MASK,

				/* SAS_MODE value may be
				 * dictated (in hw) by values
				 * of SATA_TARGET & AUTO_DET
				 */

	/* MVS_TX_CFG */
	TX_EN			= (1U << 16),	/* Enable TX */
	TX_RING_SZ_MASK		= 0xfff,	/* TX ring size, bits 11:0 */

	/* MVS_RX_CFG */
	RX_EN			= (1U << 16),	/* Enable RX */
	RX_RING_SZ_MASK		= 0xfff,	/* RX ring size, bits 11:0 */

	/* MVS_INT_COAL */
	COAL_EN			= (1U << 16),	/* Enable int coalescing */

	/* MVS_INT_STAT, MVS_INT_MASK */
	CINT_I2C		= (1U << 31),	/* I2C event */
	CINT_SW0		= (1U << 30),	/* software event 0 */
	CINT_SW1		= (1U << 29),	/* software event 1 */
	CINT_PRD_BC		= (1U << 28),	/* PRD BC err for read cmd */
	CINT_DMA_PCIE		= (1U << 27),	/* DMA to PCIE timeout */
	CINT_MEM		= (1U << 26),	/* int mem parity err */
	CINT_I2C_SLAVE		= (1U << 25),	/* slave I2C event */
	CINT_SRS		= (1U << 3),	/* SRS event */
	CINT_CI_STOP		= (1U << 1),	/* cmd issue stopped */
	CINT_DONE		= (1U << 0),	/* cmd completion */

						/* shl for ports 1-3 */
	CINT_PORT_STOPPED	= (1U << 16),	/* port0 stopped */
	CINT_PORT		= (1U << 8),	/* port0 event */
	CINT_PORT_MASK_OFFSET	= 8,
	CINT_PORT_MASK		= (0xFF << CINT_PORT_MASK_OFFSET),

	/* TX (delivery) ring bits */
	TXQ_CMD_SHIFT		= 29,
	TXQ_CMD_SSP		= 1,		/* SSP protocol */
	TXQ_CMD_SMP		= 2,		/* SMP protocol */
	TXQ_CMD_STP		= 3,		/* STP/SATA protocol */
	TXQ_CMD_SSP_FREE_LIST	= 4,		/* add to SSP targ free list */
	TXQ_CMD_SLOT_RESET	= 7,		/* reset command slot */
	TXQ_MODE_I		= (1U << 28),	/* mode: 0=target,1=initiator */
	TXQ_PRIO_HI		= (1U << 27),	/* priority: 0=normal, 1=high */
	TXQ_SRS_SHIFT		= 20,		/* SATA register set */
	TXQ_SRS_MASK		= 0x7f,
	TXQ_PHY_SHIFT		= 12,		/* PHY bitmap */
	TXQ_PHY_MASK		= 0xff,
	TXQ_SLOT_MASK		= 0xfff,	/* slot number */

	/* RX (completion) ring bits */
	RXQ_GOOD		= (1U << 23),	/* Response good */
	RXQ_SLOT_RESET		= (1U << 21),	/* Slot reset complete */
	RXQ_CMD_RX		= (1U << 20),	/* target cmd received */
	RXQ_ATTN		= (1U << 19),	/* attention */
	RXQ_RSP			= (1U << 18),	/* response frame xfer'd */
	RXQ_ERR			= (1U << 17),	/* err info rec xfer'd */
	RXQ_DONE		= (1U << 16),	/* cmd complete */
	RXQ_SLOT_MASK		= 0xfff,	/* slot number */

	/* mvs_cmd_hdr bits */
	MCH_PRD_LEN_SHIFT	= 16,		/* 16-bit PRD table len */
	MCH_SSP_FR_TYPE_SHIFT	= 13,		/* SSP frame type */

						/* SSP initiator only */
	MCH_SSP_FR_CMD		= 0x0,		/* COMMAND frame */

						/* SSP initiator or target */
	MCH_SSP_FR_TASK		= 0x1,		/* TASK frame */

						/* SSP target only */
	MCH_SSP_FR_XFER_RDY	= 0x4,		/* XFER_RDY frame */
	MCH_SSP_FR_RESP		= 0x5,		/* RESPONSE frame */
	MCH_SSP_FR_READ		= 0x6,		/* Read DATA frame(s) */
	MCH_SSP_FR_READ_RESP	= 0x7,		/* ditto, plus RESPONSE */

	MCH_PASSTHRU		= (1U << 12),	/* pass-through (SSP) */
	MCH_FBURST		= (1U << 11),	/* first burst (SSP) */
	MCH_CHK_LEN		= (1U << 10),	/* chk xfer len (SSP) */
	MCH_RETRY		= (1U << 9),	/* tport layer retry (SSP) */
	MCH_PROTECTION		= (1U << 8),	/* protection info rec (SSP) */
	MCH_RESET		= (1U << 7),	/* Reset (STP/SATA) */
	MCH_FPDMA		= (1U << 6),	/* First party DMA (STP/SATA) */
	MCH_ATAPI		= (1U << 5),	/* ATAPI (STP/SATA) */
	MCH_BIST		= (1U << 4),	/* BIST activate (STP/SATA) */
	MCH_PMP_MASK		= 0xf,		/* PMP from cmd FIS (STP/SATA)*/

	CCTL_RST		= (1U << 5),	/* port logic reset */

						/* 0(LSB first), 1(MSB first) */
	CCTL_ENDIAN_DATA	= (1U << 3),	/* PRD data */
	CCTL_ENDIAN_RSP		= (1U << 2),	/* response frame */
	CCTL_ENDIAN_OPEN	= (1U << 1),	/* open address frame */
	CCTL_ENDIAN_CMD		= (1U << 0),	/* command table */

	/* MVS_Px_SER_CTLSTAT (per-phy control) */
	PHY_SSP_RST		= (1U << 3),	/* reset SSP link layer */
	PHY_BCAST_CHG		= (1U << 2),	/* broadcast(change) notif */
	PHY_RST_HARD		= (1U << 1),	/* hard reset + phy reset */
	PHY_RST			= (1U << 0),	/* phy reset */
	PHY_MIN_SPP_PHYS_LINK_RATE_MASK = (0xF << 8),
	PHY_MAX_SPP_PHYS_LINK_RATE_MASK = (0xF << 12),
	PHY_NEG_SPP_PHYS_LINK_RATE_MASK_OFFSET = (16),
	PHY_NEG_SPP_PHYS_LINK_RATE_MASK =
			(0xF << PHY_NEG_SPP_PHYS_LINK_RATE_MASK_OFFSET),
	PHY_READY_MASK		= (1U << 20),

	/* MVS_Px_INT_STAT, MVS_Px_INT_MASK (per-phy events) */
	PHYEV_DEC_ERR		= (1U << 24),	/* Phy Decoding Error */
	PHYEV_UNASSOC_FIS	= (1U << 19),	/* unassociated FIS rx'd */
	PHYEV_AN		= (1U << 18),	/* SATA async notification */
	PHYEV_BIST_ACT		= (1U << 17),	/* BIST activate FIS */
	PHYEV_SIG_FIS		= (1U << 16),	/* signature FIS */
	PHYEV_POOF		= (1U << 12),	/* phy ready from 1 -> 0 */
	PHYEV_IU_BIG		= (1U << 11),	/* IU too long err */
	PHYEV_IU_SMALL		= (1U << 10),	/* IU too short err */
	PHYEV_UNK_TAG		= (1U << 9),	/* unknown tag */
	PHYEV_BROAD_CH		= (1U << 8),	/* broadcast(CHANGE) */
	PHYEV_COMWAKE		= (1U << 7),	/* COMWAKE rx'd */
	PHYEV_PORT_SEL		= (1U << 6),	/* port selector present */
	PHYEV_HARD_RST		= (1U << 5),	/* hard reset rx'd */
	PHYEV_ID_TMOUT		= (1U << 4),	/* identify timeout */
	PHYEV_ID_FAIL		= (1U << 3),	/* identify failed */
	PHYEV_ID_DONE		= (1U << 2),	/* identify done */
	PHYEV_HARD_RST_DONE	= (1U << 1),	/* hard reset done */
	PHYEV_RDY_CH		= (1U << 0),	/* phy ready changed state */

	/* MVS_PCS */
	PCS_EN_SATA_REG_SHIFT	= (16),		/* Enable SATA Register Set */
	PCS_EN_PORT_XMT_SHIFT	= (12),		/* Enable Port Transmit */
	PCS_EN_PORT_XMT_SHIFT2	= (8),		/* For 6480 */
	PCS_SATA_RETRY		= (1U << 8),	/* retry ctl FIS on R_ERR */
	PCS_RSP_RX_EN		= (1U << 7),	/* raw response rx */
	PCS_SELF_CLEAR		= (1U << 5),	/* self-clearing int mode */
	PCS_FIS_RX_EN		= (1U << 4),	/* FIS rx enable */
	PCS_CMD_STOP_ERR	= (1U << 3),	/* cmd stop-on-err enable */
	PCS_CMD_RST		= (1U << 1),	/* reset cmd issue */
	PCS_CMD_EN		= (1U << 0),	/* enable cmd issue */

	/* Port n Attached Device Info */
	PORT_DEV_SSP_TRGT	= (1U << 19),
	PORT_DEV_SMP_TRGT	= (1U << 18),
	PORT_DEV_STP_TRGT	= (1U << 17),
	PORT_DEV_SSP_INIT	= (1U << 11),
	PORT_DEV_SMP_INIT	= (1U << 10),
	PORT_DEV_STP_INIT	= (1U << 9),
	PORT_PHY_ID_MASK	= (0xFFU << 24),
	PORT_DEV_TRGT_MASK	= (0x7U << 17),
	PORT_DEV_INIT_MASK	= (0x7U << 9),
	PORT_DEV_TYPE_MASK	= (0x7U << 0),

	/* Port n PHY Status */
	PHY_RDY			= (1U << 2),
	PHY_DW_SYNC		= (1U << 1),
	PHY_OOB_DTCTD		= (1U << 0),

	/* VSR */
	/* PHYMODE 6 (CDB) */
	PHY_MODE6_LATECLK	= (1U << 29),	/* Lock Clock */
	PHY_MODE6_DTL_SPEED	= (1U << 27),	/* Digital Loop Speed */
	PHY_MODE6_FC_ORDER	= (1U << 26),	/* Fibre Channel Mode Order*/
	PHY_MODE6_MUCNT_EN	= (1U << 24),	/* u Count Enable */
	PHY_MODE6_SEL_MUCNT_LEN	= (1U << 22),	/* Training Length Select */
	PHY_MODE6_SELMUPI	= (1U << 20),	/* Phase Multi Select (init) */
	PHY_MODE6_SELMUPF	= (1U << 18),	/* Phase Multi Select (final) */
	PHY_MODE6_SELMUFF	= (1U << 16),	/* Freq Loop Multi Sel(final) */
	PHY_MODE6_SELMUFI	= (1U << 14),	/* Freq Loop Multi Sel(init) */
	PHY_MODE6_FREEZE_LOOP	= (1U << 12),	/* Freeze Rx CDR Loop */
	PHY_MODE6_INT_RXFOFFS	= (1U << 3),	/* Rx CDR Freq Loop Enable */
	PHY_MODE6_FRC_RXFOFFS	= (1U << 2),	/* Initial Rx CDR Offset */
	PHY_MODE6_STAU_0D8	= (1U << 1),	/* Rx CDR Freq Loop Saturate */
	PHY_MODE6_RXSAT_DIS	= (1U << 0),	/* Saturate Ctl */
};

enum mvs_info_flags {
	MVF_MSI			= (1U << 0),	/* MSI is enabled */
	MVF_PHY_PWR_FIX		= (1U << 1),	/* bug workaround */
};

enum sas_cmd_port_registers {
	CMD_CMRST_OOB_DET	= 0x100, /* COMRESET OOB detect register */
	CMD_CMWK_OOB_DET	= 0x104, /* COMWAKE OOB detect register */
	CMD_CMSAS_OOB_DET	= 0x108, /* COMSAS OOB detect register */
	CMD_BRST_OOB_DET	= 0x10c, /* burst OOB detect register */
	CMD_OOB_SPACE		= 0x110, /* OOB space control register */
	CMD_OOB_BURST		= 0x114, /* OOB burst control register */
	CMD_PHY_TIMER		= 0x118, /* PHY timer control register */
	CMD_PHY_CONFIG0		= 0x11c, /* PHY config register 0 */
	CMD_PHY_CONFIG1		= 0x120, /* PHY config register 1 */
	CMD_SAS_CTL0		= 0x124, /* SAS control register 0 */
	CMD_SAS_CTL1		= 0x128, /* SAS control register 1 */
	CMD_SAS_CTL2		= 0x12c, /* SAS control register 2 */
	CMD_SAS_CTL3		= 0x130, /* SAS control register 3 */
	CMD_ID_TEST		= 0x134, /* ID test register */
	CMD_PL_TIMER		= 0x138, /* PL timer register */
	CMD_WD_TIMER		= 0x13c, /* WD timer register */
	CMD_PORT_SEL_COUNT	= 0x140, /* port selector count register */
	CMD_APP_MEM_CTL		= 0x144, /* Application Memory Control */
	CMD_XOR_MEM_CTL		= 0x148, /* XOR Block Memory Control */
	CMD_DMA_MEM_CTL		= 0x14c, /* DMA Block Memory Control */
	CMD_PORT_MEM_CTL0	= 0x150, /* Port Memory Control 0 */
	CMD_PORT_MEM_CTL1	= 0x154, /* Port Memory Control 1 */
	CMD_SATA_PORT_MEM_CTL0	= 0x158, /* SATA Port Memory Control 0 */
	CMD_SATA_PORT_MEM_CTL1	= 0x15c, /* SATA Port Memory Control 1 */
	CMD_XOR_MEM_BIST_CTL	= 0x160, /* XOR Memory BIST Control */
	CMD_XOR_MEM_BIST_STAT	= 0x164, /* XOR Memroy BIST Status */
	CMD_DMA_MEM_BIST_CTL	= 0x168, /* DMA Memory BIST Control */
	CMD_DMA_MEM_BIST_STAT	= 0x16c, /* DMA Memory BIST Status */
	CMD_PORT_MEM_BIST_CTL	= 0x170, /* Port Memory BIST Control */
	CMD_PORT_MEM_BIST_STAT0 = 0x174, /* Port Memory BIST Status 0 */
	CMD_PORT_MEM_BIST_STAT1 = 0x178, /* Port Memory BIST Status 1 */
	CMD_STP_MEM_BIST_CTL	= 0x17c, /* STP Memory BIST Control */
	CMD_STP_MEM_BIST_STAT0	= 0x180, /* STP Memory BIST Status 0 */
	CMD_STP_MEM_BIST_STAT1	= 0x184, /* STP Memory BIST Status 1 */
	CMD_RESET_COUNT		= 0x188, /* Reset Count */
	CMD_MONTR_DATA_SEL	= 0x18C, /* Monitor Data/Select */
	CMD_PLL_PHY_CONFIG	= 0x190, /* PLL/PHY Configuration */
	CMD_PHY_CTL		= 0x194, /* PHY Control and Status */
	CMD_PHY_TEST_COUNT0	= 0x198, /* Phy Test Count 0 */
	CMD_PHY_TEST_COUNT1	= 0x19C, /* Phy Test Count 1 */
	CMD_PHY_TEST_COUNT2	= 0x1A0, /* Phy Test Count 2 */
	CMD_APP_ERR_CONFIG	= 0x1A4, /* Application Error Configuration */
	CMD_PND_FIFO_CTL0	= 0x1A8, /* Pending FIFO Control 0 */
	CMD_HOST_CTL		= 0x1AC, /* Host Control Status */
	CMD_HOST_WR_DATA	= 0x1B0, /* Host Write Data */
	CMD_HOST_RD_DATA	= 0x1B4, /* Host Read Data */
	CMD_PHY_MODE_21		= 0x1B8, /* Phy Mode 21 */
	CMD_SL_MODE0		= 0x1BC, /* SL Mode 0 */
	CMD_SL_MODE1		= 0x1C0, /* SL Mode 1 */
	CMD_PND_FIFO_CTL1	= 0x1C4, /* Pending FIFO Control 1 */
};

/* SAS/SATA configuration port registers, aka phy registers */
enum sas_sata_config_port_regs {
	PHYR_IDENTIFY		= 0x00,	/* info for IDENTIFY frame */
	PHYR_ADDR_LO		= 0x04,	/* my SAS address (low) */
	PHYR_ADDR_HI		= 0x08,	/* my SAS address (high) */
	PHYR_ATT_DEV_INFO	= 0x0C,	/* attached device info */
	PHYR_ATT_ADDR_LO	= 0x10,	/* attached dev SAS addr (low) */
	PHYR_ATT_ADDR_HI	= 0x14,	/* attached dev SAS addr (high) */
	PHYR_SATA_CTL		= 0x18,	/* SATA control */
	PHYR_PHY_STAT		= 0x1C,	/* PHY status */
	PHYR_SATA_SIG0		= 0x20,	/*port SATA signature FIS(Byte 0-3) */
	PHYR_SATA_SIG1		= 0x24,	/*port SATA signature FIS(Byte 4-7) */
	PHYR_SATA_SIG2		= 0x28,	/*port SATA signature FIS(Byte 8-11) */
	PHYR_SATA_SIG3		= 0x2c,	/*port SATA signature FIS(Byte 12-15) */
	PHYR_R_ERR_COUNT	= 0x30, /* port R_ERR count register */
	PHYR_CRC_ERR_COUNT	= 0x34, /* port CRC error count register */
	PHYR_WIDE_PORT		= 0x38,	/* wide port participating */
	PHYR_CURRENT0		= 0x80,	/* current connection info 0 */
	PHYR_CURRENT1		= 0x84,	/* current connection info 1 */
	PHYR_CURRENT2		= 0x88,	/* current connection info 2 */
};

/*  SAS/SATA Vendor Specific Port Registers */
enum sas_sata_vsp_regs {
	VSR_PHY_STAT		= 0x00, /* Phy Status */
	VSR_PHY_MODE1		= 0x01, /* phy tx */
	VSR_PHY_MODE2		= 0x02, /* tx scc */
	VSR_PHY_MODE3		= 0x03, /* pll */
	VSR_PHY_MODE4		= 0x04, /* VCO */
	VSR_PHY_MODE5		= 0x05, /* Rx */
	VSR_PHY_MODE6		= 0x06, /* CDR */
	VSR_PHY_MODE7		= 0x07, /* Impedance */
	VSR_PHY_MODE8		= 0x08, /* Voltage */
	VSR_PHY_MODE9		= 0x09, /* Test */
	VSR_PHY_MODE10		= 0x0A, /* Power */
	VSR_PHY_MODE11		= 0x0B, /* Phy Mode */
	VSR_PHY_VS0		= 0x0C, /* Vednor Specific 0 */
	VSR_PHY_VS1		= 0x0D, /* Vednor Specific 1 */
};

enum pci_cfg_registers {
	PCR_PHY_CTL	= 0x40,
	PCR_PHY_CTL2	= 0x90,
	PCR_DEV_CTRL	= 0xE8,
};

enum pci_cfg_register_bits {
	PCTL_PWR_ON	= (0xFU << 24),
	PCTL_OFF	= (0xFU << 12),
	PRD_REQ_SIZE	= (0x4000),
	PRD_REQ_MASK	= (0x00007000),
};

enum nvram_layout_offsets {
	NVR_SIG		= 0x00,		/* 0xAA, 0x55 */
	NVR_SAS_ADDR	= 0x02,		/* 8-byte SAS address */
};

enum chip_flavors {
	chip_6320,
	chip_6440,
	chip_6480,
};

enum port_type {
	PORT_TYPE_SAS	=  (1L << 1),
	PORT_TYPE_SATA	=  (1L << 0),
};

/* Command Table Format */
enum ct_format {
	/* SSP */
	SSP_F_H		=  0x00,
	SSP_F_IU	=  0x18,
	SSP_F_MAX	=  0x4D,
	/* STP */
	STP_CMD_FIS	=  0x00,
	STP_ATAPI_CMD	=  0x40,
	STP_F_MAX	=  0x10,
	/* SMP */
	SMP_F_T		=  0x00,
	SMP_F_DEP	=  0x01,
	SMP_F_MAX	=  0x101,
};

enum status_buffer {
	SB_EIR_OFF	=  0x00,	/* Error Information Record */
	SB_RFB_OFF	=  0x08,	/* Response Frame Buffer */
	SB_RFB_MAX	=  0x400,	/* RFB size*/
};

enum error_info_rec {
	CMD_ISS_STPD	= (1U << 31),	/* Cmd Issue Stopped */
	CMD_PI_ERR	= (1U << 30),	/* Protection info error.  see flags2 */
	RSP_OVER	= (1U << 29),	/* rsp buffer overflow */
	RETRY_LIM	= (1U << 28),	/* FIS/frame retry limit exceeded */
	UNK_FIS 	= (1U << 27),	/* unknown FIS */
	DMA_TERM	= (1U << 26),	/* DMA terminate primitive rx'd */
	SYNC_ERR	= (1U << 25),	/* SYNC rx'd during frame xmit */
	TFILE_ERR	= (1U << 24),	/* SATA taskfile Error bit set */
	R_ERR		= (1U << 23),	/* SATA returned R_ERR prim */
	RD_OFS		= (1U << 20),	/* Read DATA frame invalid offset */
	XFER_RDY_OFS	= (1U << 19),	/* XFER_RDY offset error */
	UNEXP_XFER_RDY	= (1U << 18),	/* unexpected XFER_RDY error */
	DATA_OVER_UNDER = (1U << 16),	/* data overflow/underflow */
	INTERLOCK	= (1U << 15),	/* interlock error */
	NAK		= (1U << 14),	/* NAK rx'd */
	ACK_NAK_TO	= (1U << 13),	/* ACK/NAK timeout */
	CXN_CLOSED	= (1U << 12),	/* cxn closed w/out ack/nak */
	OPEN_TO 	= (1U << 11),	/* I_T nexus lost, open cxn timeout */
	PATH_BLOCKED	= (1U << 10),	/* I_T nexus lost, pathway blocked */
	NO_DEST 	= (1U << 9),	/* I_T nexus lost, no destination */
	STP_RES_BSY	= (1U << 8),	/* STP resources busy */
	BREAK		= (1U << 7),	/* break received */
	BAD_DEST	= (1U << 6),	/* bad destination */
	BAD_PROTO	= (1U << 5),	/* protocol not supported */
	BAD_RATE	= (1U << 4),	/* cxn rate not supported */
	WRONG_DEST	= (1U << 3),	/* wrong destination error */
	CREDIT_TO	= (1U << 2),	/* credit timeout */
	WDOG_TO 	= (1U << 1),	/* watchdog timeout */
	BUF_PAR 	= (1U << 0),	/* buffer parity error */
};

enum error_info_rec_2 {
	SLOT_BSY_ERR	= (1U << 31),	/* Slot Busy Error */
	GRD_CHK_ERR	= (1U << 14),	/* Guard Check Error */
	APP_CHK_ERR	= (1U << 13),	/* Application Check error */
	REF_CHK_ERR	= (1U << 12),	/* Reference Check Error */
	USR_BLK_NM	= (1U << 0),	/* User Block Number */
};

struct mvs_chip_info {
	u32		n_phy;
	u32		srs_sz;
	u32		slot_width;
};

struct mvs_err_info {
	__le32			flags;
	__le32			flags2;
};

struct mvs_prd {
	__le64			addr;		/* 64-bit buffer address */
	__le32			reserved;
	__le32			len;		/* 16-bit length */
};

struct mvs_cmd_hdr {
	__le32			flags;		/* PRD tbl len; SAS, SATA ctl */
	__le32			lens;		/* cmd, max resp frame len */
	__le32			tags;		/* targ port xfer tag; tag */
	__le32			data_len;	/* data xfer len */
	__le64			cmd_tbl;	/* command table address */
	__le64			open_frame;	/* open addr frame address */
	__le64			status_buf;	/* status buffer address */
	__le64			prd_tbl;	/* PRD tbl address */
	__le32			reserved[4];
};

struct mvs_port {
	struct asd_sas_port	sas_port;
	u8			port_attached;
	u8			taskfileset;
	u8			wide_port_phymap;
	struct list_head	list;
};

struct mvs_phy {
	struct mvs_port		*port;
	struct asd_sas_phy	sas_phy;
	struct sas_identify	identify;
	struct scsi_device	*sdev;
	u64		dev_sas_addr;
	u64		att_dev_sas_addr;
	u32		att_dev_info;
	u32		dev_info;
	u32		phy_type;
	u32		phy_status;
	u32		irq_status;
	u32		frame_rcvd_size;
	u8		frame_rcvd[32];
	u8		phy_attached;
	enum sas_linkrate	minimum_linkrate;
	enum sas_linkrate	maximum_linkrate;
};

struct mvs_slot_info {
	struct list_head	list;
	struct sas_task		*task;
	u32			n_elem;
	u32			tx;

	/* DMA buffer for storing cmd tbl, open addr frame, status buffer,
	 * and PRD table
	 */
	void			*buf;
	dma_addr_t		buf_dma;
#if _MV_DUMP
	u32			cmd_size;
#endif

	void			*response;
	struct mvs_port		*port;
};

struct mvs_info {
	unsigned long		flags;

	spinlock_t		lock;		/* host-wide lock */
	struct pci_dev		*pdev;		/* our device */
	void __iomem		*regs;		/* enhanced mode registers */
	void __iomem		*peri_regs;	/* peripheral registers */

	u8			sas_addr[SAS_ADDR_SIZE];
	struct sas_ha_struct	sas;		/* SCSI/SAS glue */
	struct Scsi_Host	*shost;

	__le32			*tx;		/* TX (delivery) DMA ring */
	dma_addr_t		tx_dma;
	u32			tx_prod;	/* cached next-producer idx */

	__le32			*rx;		/* RX (completion) DMA ring */
	dma_addr_t		rx_dma;
	u32			rx_cons;	/* RX consumer idx */

	__le32			*rx_fis;	/* RX'd FIS area */
	dma_addr_t		rx_fis_dma;

	struct mvs_cmd_hdr	*slot;	/* DMA command header slots */
	dma_addr_t		slot_dma;

	const struct mvs_chip_info *chip;

	u8			tags[MVS_SLOTS];
	struct mvs_slot_info	slot_info[MVS_SLOTS];
				/* further per-slot information */
	struct mvs_phy		phy[MVS_MAX_PHYS];
	struct mvs_port		port[MVS_MAX_PHYS];
#ifdef MVS_USE_TASKLET
	struct tasklet_struct	tasklet;
#endif
};

static int mvs_phy_control(struct asd_sas_phy *sas_phy, enum phy_func func,
			   void *funcdata);
static u32 mvs_read_phy_ctl(struct mvs_info *mvi, u32 port);
static void mvs_write_phy_ctl(struct mvs_info *mvi, u32 port, u32 val);
static u32 mvs_read_port_irq_stat(struct mvs_info *mvi, u32 port);
static void mvs_write_port_irq_stat(struct mvs_info *mvi, u32 port, u32 val);
static void mvs_write_port_irq_mask(struct mvs_info *mvi, u32 port, u32 val);
static u32 mvs_read_port_irq_mask(struct mvs_info *mvi, u32 port);

static u32 mvs_is_phy_ready(struct mvs_info *mvi, int i);
static void mvs_detect_porttype(struct mvs_info *mvi, int i);
static void mvs_update_phyinfo(struct mvs_info *mvi, int i, int get_st);
static void mvs_release_task(struct mvs_info *mvi, int phy_no);

static int mvs_scan_finished(struct Scsi_Host *, unsigned long);
static void mvs_scan_start(struct Scsi_Host *);
static int mvs_slave_configure(struct scsi_device *sdev);

static struct scsi_transport_template *mvs_stt;

static const struct mvs_chip_info mvs_chips[] = {
	[chip_6320] =		{ 2, 16, 9  },
	[chip_6440] =		{ 4, 16, 9  },
	[chip_6480] =		{ 8, 32, 10 },
};

static struct scsi_host_template mvs_sht = {
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.queuecommand		= sas_queuecommand,
	.target_alloc		= sas_target_alloc,
	.slave_configure	= mvs_slave_configure,
	.slave_destroy		= sas_slave_destroy,
	.scan_finished		= mvs_scan_finished,
	.scan_start		= mvs_scan_start,
	.change_queue_depth	= sas_change_queue_depth,
	.change_queue_type	= sas_change_queue_type,
	.bios_param		= sas_bios_param,
	.can_queue		= 1,
	.cmd_per_lun		= 1,
	.this_id		= -1,
	.sg_tablesize		= SG_ALL,
	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
	.use_clustering		= ENABLE_CLUSTERING,
	.eh_device_reset_handler	= sas_eh_device_reset_handler,
	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
	.slave_alloc		= sas_slave_alloc,
	.target_destroy		= sas_target_destroy,
	.ioctl			= sas_ioctl,
};

static void mvs_hexdump(u32 size, u8 *data, u32 baseaddr)
{
	u32 i;
	u32 run;
	u32 offset;

	offset = 0;
	while (size) {
		printk("%08X : ", baseaddr + offset);
		if (size >= 16)
			run = 16;
		else
			run = size;
		size -= run;
		for (i = 0; i < 16; i++) {
			if (i < run)
				printk("%02X ", (u32)data[i]);
			else
				printk("   ");
		}
		printk(": ");
		for (i = 0; i < run; i++)
			printk("%c", isalnum(data[i]) ? data[i] : '.');
		printk("\n");
		data = &data[16];
		offset += run;
	}
	printk("\n");
}

#if _MV_DUMP
static void mvs_hba_sb_dump(struct mvs_info *mvi, u32 tag,
				   enum sas_protocol proto)
{
	u32 offset;
	struct pci_dev *pdev = mvi->pdev;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];

	offset = slot->cmd_size + MVS_OAF_SZ +
	    sizeof(struct mvs_prd) * slot->n_elem;
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Status buffer[%d] :\n",
			tag);
	mvs_hexdump(32, (u8 *) slot->response,
		    (u32) slot->buf_dma + offset);
}
#endif

static void mvs_hba_memory_dump(struct mvs_info *mvi, u32 tag,
				enum sas_protocol proto)
{
#if _MV_DUMP
	u32 sz, w_ptr;
	u64 addr;
	void __iomem *regs = mvi->regs;
	struct pci_dev *pdev = mvi->pdev;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];

	/*Delivery Queue */
	sz = mr32(TX_CFG) & TX_RING_SZ_MASK;
	w_ptr = slot->tx;
	addr = mr32(TX_HI) << 16 << 16 | mr32(TX_LO);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Delivery Queue Size=%04d , WRT_PTR=%04X\n", sz, w_ptr);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Delivery Queue Base Address=0x%llX (PA)"
		"(tx_dma=0x%llX), Entry=%04d\n",
		addr, mvi->tx_dma, w_ptr);
	mvs_hexdump(sizeof(u32), (u8 *)(&mvi->tx[mvi->tx_prod]),
			(u32) mvi->tx_dma + sizeof(u32) * w_ptr);
	/*Command List */
	addr = mvi->slot_dma;
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Command List Base Address=0x%llX (PA)"
		"(slot_dma=0x%llX), Header=%03d\n",
		addr, slot->buf_dma, tag);
	dev_printk(KERN_DEBUG, &pdev->dev, "Command Header[%03d]:\n", tag);
	/*mvs_cmd_hdr */
	mvs_hexdump(sizeof(struct mvs_cmd_hdr), (u8 *)(&mvi->slot[tag]),
		(u32) mvi->slot_dma + tag * sizeof(struct mvs_cmd_hdr));
	/*1.command table area */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Command Table :\n");
	mvs_hexdump(slot->cmd_size, (u8 *) slot->buf, (u32) slot->buf_dma);
	/*2.open address frame area */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->Open Address Frame :\n");
	mvs_hexdump(MVS_OAF_SZ, (u8 *) slot->buf + slot->cmd_size,
				(u32) slot->buf_dma + slot->cmd_size);
	/*3.status buffer */
	mvs_hba_sb_dump(mvi, tag, proto);
	/*4.PRD table */
	dev_printk(KERN_DEBUG, &pdev->dev, "+---->PRD table :\n");
	mvs_hexdump(sizeof(struct mvs_prd) * slot->n_elem,
		(u8 *) slot->buf + slot->cmd_size + MVS_OAF_SZ,
		(u32) slot->buf_dma + slot->cmd_size + MVS_OAF_SZ);
#endif
}

static void mvs_hba_cq_dump(struct mvs_info *mvi)
{
#if (_MV_DUMP > 2)
	u64 addr;
	void __iomem *regs = mvi->regs;
	struct pci_dev *pdev = mvi->pdev;
	u32 entry = mvi->rx_cons + 1;
	u32 rx_desc = le32_to_cpu(mvi->rx[entry]);

	/*Completion Queue */
	addr = mr32(RX_HI) << 16 << 16 | mr32(RX_LO);
	dev_printk(KERN_DEBUG, &pdev->dev, "Completion Task = 0x%p\n",
		   mvi->slot_info[rx_desc & RXQ_SLOT_MASK].task);
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Completion List Base Address=0x%llX (PA), "
		"CQ_Entry=%04d, CQ_WP=0x%08X\n",
		addr, entry - 1, mvi->rx[0]);
	mvs_hexdump(sizeof(u32), (u8 *)(&rx_desc),
		    mvi->rx_dma + sizeof(u32) * entry);
#endif
}

static void mvs_hba_interrupt_enable(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp;

	tmp = mr32(GBL_CTL);

	mw32(GBL_CTL, tmp | INT_EN);
}

static void mvs_hba_interrupt_disable(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp;

	tmp = mr32(GBL_CTL);

	mw32(GBL_CTL, tmp & ~INT_EN);
}

static int mvs_int_rx(struct mvs_info *mvi, bool self_clear);

/* move to PCI layer or libata core? */
static int pci_go_64(struct pci_dev *pdev)
{
	int rc;

	if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
		if (rc) {
			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
			if (rc) {
				dev_printk(KERN_ERR, &pdev->dev,
					   "64-bit DMA enable failed\n");
				return rc;
			}
		}
	} else {
		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit DMA enable failed\n");
			return rc;
		}
		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit consistent DMA enable failed\n");
			return rc;
		}
	}

	return rc;
}

static int mvs_find_tag(struct mvs_info *mvi, struct sas_task *task, u32 *tag)
{
	if (task->lldd_task) {
		struct mvs_slot_info *slot;
		slot = (struct mvs_slot_info *) task->lldd_task;
		*tag = slot - mvi->slot_info;
		return 1;
	}
	return 0;
}

static void mvs_tag_clear(struct mvs_info *mvi, u32 tag)
{
	void *bitmap = (void *) &mvi->tags;
	clear_bit(tag, bitmap);
}

static void mvs_tag_free(struct mvs_info *mvi, u32 tag)
{
	mvs_tag_clear(mvi, tag);
}

static void mvs_tag_set(struct mvs_info *mvi, unsigned int tag)
{
	void *bitmap = (void *) &mvi->tags;
	set_bit(tag, bitmap);
}

static int mvs_tag_alloc(struct mvs_info *mvi, u32 *tag_out)
{
	unsigned int index, tag;
	void *bitmap = (void *) &mvi->tags;

	index = find_first_zero_bit(bitmap, MVS_SLOTS);
	tag = index;
	if (tag >= MVS_SLOTS)
		return -SAS_QUEUE_FULL;
	mvs_tag_set(mvi, tag);
	*tag_out = tag;
	return 0;
}

static void mvs_tag_init(struct mvs_info *mvi)
{
	int i;
	for (i = 0; i < MVS_SLOTS; ++i)
		mvs_tag_clear(mvi, i);
}

#ifndef MVS_DISABLE_NVRAM
static int mvs_eep_read(void __iomem *regs, u32 addr, u32 *data)
{
	int timeout = 1000;

	if (addr & ~SPI_ADDR_MASK)
		return -EINVAL;

	writel(addr, regs + SPI_CMD);
	writel(TWSI_RD, regs + SPI_CTL);

	while (timeout-- > 0) {
		if (readl(regs + SPI_CTL) & TWSI_RDY) {
			*data = readl(regs + SPI_DATA);
			return 0;
		}

		udelay(10);
	}

	return -EBUSY;
}

static int mvs_eep_read_buf(void __iomem *regs, u32 addr,
			    void *buf, u32 buflen)
{
	u32 addr_end, tmp_addr, i, j;
	u32 tmp = 0;
	int rc;
	u8 *tmp8, *buf8 = buf;

	addr_end = addr + buflen;
	tmp_addr = ALIGN(addr, 4);
	if (addr > 0xff)
		return -EINVAL;

	j = addr & 0x3;
	if (j) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		tmp8 = (u8 *)&tmp;
		for (i = j; i < 4; i++)
			*buf8++ = tmp8[i];

		tmp_addr += 4;
	}

	for (j = ALIGN(addr_end, 4); tmp_addr < j; tmp_addr += 4) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		memcpy(buf8, &tmp, 4);
		buf8 += 4;
	}

	if (tmp_addr < addr_end) {
		rc = mvs_eep_read(regs, tmp_addr, &tmp);
		if (rc)
			return rc;

		tmp8 = (u8 *)&tmp;
		j = addr_end - tmp_addr;
		for (i = 0; i < j; i++)
			*buf8++ = tmp8[i];

		tmp_addr += 4;
	}

	return 0;
}
#endif

static int mvs_nvram_read(struct mvs_info *mvi, u32 addr,
			  void *buf, u32 buflen)
{
#ifndef MVS_DISABLE_NVRAM
	void __iomem *regs = mvi->regs;
	int rc, i;
	u32 sum;
	u8 hdr[2], *tmp;
	const char *msg;

	rc = mvs_eep_read_buf(regs, addr, &hdr, 2);
	if (rc) {
		msg = "nvram hdr read failed";
		goto err_out;
	}
	rc = mvs_eep_read_buf(regs, addr + 2, buf, buflen);
	if (rc) {
		msg = "nvram read failed";
		goto err_out;
	}

	if (hdr[0] != 0x5A) {
		/* entry id */
		msg = "invalid nvram entry id";
		rc = -ENOENT;
		goto err_out;
	}

	tmp = buf;
	sum = ((u32)hdr[0]) + ((u32)hdr[1]);
	for (i = 0; i < buflen; i++)
		sum += ((u32)tmp[i]);

	if (sum) {
		msg = "nvram checksum failure";
		rc = -EILSEQ;
		goto err_out;
	}

	return 0;

err_out:
	dev_printk(KERN_ERR, &mvi->pdev->dev, "%s", msg);
	return rc;
#else
	/* FIXME , For SAS target mode */
	memcpy(buf, "\x50\x05\x04\x30\x11\xab\x00\x00", 8);
	return 0;
#endif
}

static void mvs_bytes_dmaed(struct mvs_info *mvi, int i)
{
	struct mvs_phy *phy = &mvi->phy[i];
	struct asd_sas_phy *sas_phy = mvi->sas.sas_phy[i];

	if (!phy->phy_attached)
		return;

	if (sas_phy->phy) {
		struct sas_phy *sphy = sas_phy->phy;

		sphy->negotiated_linkrate = sas_phy->linkrate;
		sphy->minimum_linkrate = phy->minimum_linkrate;
		sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS;
		sphy->maximum_linkrate = phy->maximum_linkrate;
		sphy->maximum_linkrate_hw = SAS_LINK_RATE_3_0_GBPS;
	}

	if (phy->phy_type & PORT_TYPE_SAS) {
		struct sas_identify_frame *id;

		id = (struct sas_identify_frame *)phy->frame_rcvd;
		id->dev_type = phy->identify.device_type;
		id->initiator_bits = SAS_PROTOCOL_ALL;
		id->target_bits = phy->identify.target_port_protocols;
	} else if (phy->phy_type & PORT_TYPE_SATA) {
		/* TODO */
	}
	mvi->sas.sas_phy[i]->frame_rcvd_size = phy->frame_rcvd_size;
	mvi->sas.notify_port_event(mvi->sas.sas_phy[i],
				   PORTE_BYTES_DMAED);
}

static int mvs_scan_finished(struct Scsi_Host *shost, unsigned long time)
{
	/* give the phy enabling interrupt event time to come in (1s
	 * is empirically about all it takes) */
	if (time < HZ)
		return 0;
	/* Wait for discovery to finish */
	scsi_flush_work(shost);
	return 1;
}

static void mvs_scan_start(struct Scsi_Host *shost)
{
	int i;
	struct mvs_info *mvi = SHOST_TO_SAS_HA(shost)->lldd_ha;

	for (i = 0; i < mvi->chip->n_phy; ++i) {
		mvs_bytes_dmaed(mvi, i);
	}
}

static int mvs_slave_configure(struct scsi_device *sdev)
{
	struct domain_device *dev = sdev_to_domain_dev(sdev);
	int ret = sas_slave_configure(sdev);

	if (ret)
		return ret;

	if (dev_is_sata(dev)) {
		/* struct ata_port *ap = dev->sata_dev.ap; */
		/* struct ata_device *adev = ap->link.device; */

		/* clamp at no NCQ for the time being */
		/* adev->flags |= ATA_DFLAG_NCQ_OFF; */
		scsi_adjust_queue_depth(sdev, MSG_SIMPLE_TAG, 1);
	}
	return 0;
}

static void mvs_int_port(struct mvs_info *mvi, int phy_no, u32 events)
{
	struct pci_dev *pdev = mvi->pdev;
	struct sas_ha_struct *sas_ha = &mvi->sas;
	struct mvs_phy *phy = &mvi->phy[phy_no];
	struct asd_sas_phy *sas_phy = &phy->sas_phy;

	phy->irq_status = mvs_read_port_irq_stat(mvi, phy_no);
	/*
	* events is port event now ,
	* we need check the interrupt status which belongs to per port.
	*/
	dev_printk(KERN_DEBUG, &pdev->dev,
		"Port %d Event = %X\n",
		phy_no, phy->irq_status);

	if (phy->irq_status & (PHYEV_POOF | PHYEV_DEC_ERR)) {
		mvs_release_task(mvi, phy_no);
		if (!mvs_is_phy_ready(mvi, phy_no)) {
			sas_phy_disconnected(sas_phy);
			sas_ha->notify_phy_event(sas_phy, PHYE_LOSS_OF_SIGNAL);
			dev_printk(KERN_INFO, &pdev->dev,
				"Port %d Unplug Notice\n", phy_no);

		} else
			mvs_phy_control(sas_phy, PHY_FUNC_LINK_RESET, NULL);
	}
	if (!(phy->irq_status & PHYEV_DEC_ERR)) {
		if (phy->irq_status & PHYEV_COMWAKE) {
			u32 tmp = mvs_read_port_irq_mask(mvi, phy_no);
			mvs_write_port_irq_mask(mvi, phy_no,
						tmp | PHYEV_SIG_FIS);
		}
		if (phy->irq_status & (PHYEV_SIG_FIS | PHYEV_ID_DONE)) {
			phy->phy_status = mvs_is_phy_ready(mvi, phy_no);
			if (phy->phy_status) {
				mvs_detect_porttype(mvi, phy_no);

				if (phy->phy_type & PORT_TYPE_SATA) {
					u32 tmp = mvs_read_port_irq_mask(mvi,
								phy_no);
					tmp &= ~PHYEV_SIG_FIS;
					mvs_write_port_irq_mask(mvi,
								phy_no, tmp);
				}

				mvs_update_phyinfo(mvi, phy_no, 0);
				sas_ha->notify_phy_event(sas_phy,
							PHYE_OOB_DONE);
				mvs_bytes_dmaed(mvi, phy_no);
			} else {
				dev_printk(KERN_DEBUG, &pdev->dev,
					"plugin interrupt but phy is gone\n");
				mvs_phy_control(sas_phy, PHY_FUNC_LINK_RESET,
							NULL);
			}
		} else if (phy->irq_status & PHYEV_BROAD_CH) {
			mvs_release_task(mvi, phy_no);
			sas_ha->notify_port_event(sas_phy,
						PORTE_BROADCAST_RCVD);
		}
	}
	mvs_write_port_irq_stat(mvi, phy_no, phy->irq_status);
}

static void mvs_int_sata(struct mvs_info *mvi)
{
	u32 tmp;
	void __iomem *regs = mvi->regs;
	tmp = mr32(INT_STAT_SRS);
	mw32(INT_STAT_SRS, tmp & 0xFFFF);
}

static void mvs_slot_reset(struct mvs_info *mvi, struct sas_task *task,
				u32 slot_idx)
{
	void __iomem *regs = mvi->regs;
	struct domain_device *dev = task->dev;
	struct asd_sas_port *sas_port = dev->port;
	struct mvs_port *port = mvi->slot_info[slot_idx].port;
	u32 reg_set, phy_mask;

	if (!sas_protocol_ata(task->task_proto)) {
		reg_set = 0;
		phy_mask = (port->wide_port_phymap) ? port->wide_port_phymap :
				sas_port->phy_mask;
	} else {
		reg_set = port->taskfileset;
		phy_mask = sas_port->phy_mask;
	}
	mvi->tx[mvi->tx_prod] = cpu_to_le32(TXQ_MODE_I | slot_idx |
					(TXQ_CMD_SLOT_RESET << TXQ_CMD_SHIFT) |
					(phy_mask << TXQ_PHY_SHIFT) |
					(reg_set << TXQ_SRS_SHIFT));

	mw32(TX_PROD_IDX, mvi->tx_prod);
	mvi->tx_prod = (mvi->tx_prod + 1) & (MVS_CHIP_SLOT_SZ - 1);
}

static int mvs_sata_done(struct mvs_info *mvi, struct sas_task *task,
			u32 slot_idx, int err)
{
	struct mvs_port *port = mvi->slot_info[slot_idx].port;
	struct task_status_struct *tstat = &task->task_status;
	struct ata_task_resp *resp = (struct ata_task_resp *)tstat->buf;
	int stat = SAM_GOOD;

	resp->frame_len = sizeof(struct dev_to_host_fis);
	memcpy(&resp->ending_fis[0],
	       SATA_RECEIVED_D2H_FIS(port->taskfileset),
	       sizeof(struct dev_to_host_fis));
	tstat->buf_valid_size = sizeof(*resp);
	if (unlikely(err))
		stat = SAS_PROTO_RESPONSE;
	return stat;
}

static void mvs_slot_free(struct mvs_info *mvi, u32 rx_desc)
{
	u32 slot_idx = rx_desc & RXQ_SLOT_MASK;
	mvs_tag_clear(mvi, slot_idx);
}

static void mvs_slot_task_free(struct mvs_info *mvi, struct sas_task *task,
			  struct mvs_slot_info *slot, u32 slot_idx)
{
	if (!sas_protocol_ata(task->task_proto))
		if (slot->n_elem)
			pci_unmap_sg(mvi->pdev, task->scatter,
				     slot->n_elem, task->data_dir);

	switch (task->task_proto) {
	case SAS_PROTOCOL_SMP:
		pci_unmap_sg(mvi->pdev, &task->smp_task.smp_resp, 1,
			     PCI_DMA_FROMDEVICE);
		pci_unmap_sg(mvi->pdev, &task->smp_task.smp_req, 1,
			     PCI_DMA_TODEVICE);
		break;

	case SAS_PROTOCOL_SATA:
	case SAS_PROTOCOL_STP:
	case SAS_PROTOCOL_SSP:
	default:
		/* do nothing */
		break;
	}
	list_del(&slot->list);
	task->lldd_task = NULL;
	slot->task = NULL;
	slot->port = NULL;
}

static int mvs_slot_err(struct mvs_info *mvi, struct sas_task *task,
			 u32 slot_idx)
{
	struct mvs_slot_info *slot = &mvi->slot_info[slot_idx];
	u32 err_dw0 = le32_to_cpu(*(u32 *) (slot->response));
	u32 err_dw1 = le32_to_cpu(*(u32 *) (slot->response + 4));
	int stat = SAM_CHECK_COND;

	if (err_dw1 & SLOT_BSY_ERR) {
		stat = SAS_QUEUE_FULL;
		mvs_slot_reset(mvi, task, slot_idx);
	}
	switch (task->task_proto) {
	case SAS_PROTOCOL_SSP:
		break;
	case SAS_PROTOCOL_SMP:
		break;
	case SAS_PROTOCOL_SATA:
	case SAS_PROTOCOL_STP:
	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
		if (err_dw0 & TFILE_ERR)
			stat = mvs_sata_done(mvi, task, slot_idx, 1);
		break;
	default:
		break;
	}

	mvs_hexdump(16, (u8 *) slot->response, 0);
	return stat;
}

static int mvs_slot_complete(struct mvs_info *mvi, u32 rx_desc, u32 flags)
{
	u32 slot_idx = rx_desc & RXQ_SLOT_MASK;
	struct mvs_slot_info *slot = &mvi->slot_info[slot_idx];
	struct sas_task *task = slot->task;
	struct task_status_struct *tstat;
	struct mvs_port *port;
	bool aborted;
	void *to;

	if (unlikely(!task || !task->lldd_task))
		return -1;

	mvs_hba_cq_dump(mvi);

	spin_lock(&task->task_state_lock);
	aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED;
	if (!aborted) {
		task->task_state_flags &=
		    ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
		task->task_state_flags |= SAS_TASK_STATE_DONE;
	}
	spin_unlock(&task->task_state_lock);

	if (aborted) {
		mvs_slot_task_free(mvi, task, slot, slot_idx);
		mvs_slot_free(mvi, rx_desc);
		return -1;
	}

	port = slot->port;
	tstat = &task->task_status;
	memset(tstat, 0, sizeof(*tstat));
	tstat->resp = SAS_TASK_COMPLETE;

	if (unlikely(!port->port_attached || flags)) {
		mvs_slot_err(mvi, task, slot_idx);
		if (!sas_protocol_ata(task->task_proto))
			tstat->stat = SAS_PHY_DOWN;
		goto out;
	}

	/* error info record present */
	if (unlikely((rx_desc & RXQ_ERR) && (*(u64 *) slot->response))) {
		tstat->stat = mvs_slot_err(mvi, task, slot_idx);
		goto out;
	}

	switch (task->task_proto) {
	case SAS_PROTOCOL_SSP:
		/* hw says status == 0, datapres == 0 */
		if (rx_desc & RXQ_GOOD) {
			tstat->stat = SAM_GOOD;
			tstat->resp = SAS_TASK_COMPLETE;
		}
		/* response frame present */
		else if (rx_desc & RXQ_RSP) {
			struct ssp_response_iu *iu =
			    slot->response + sizeof(struct mvs_err_info);
			sas_ssp_task_response(&mvi->pdev->dev, task, iu);
		}

		/* should never happen? */
		else
			tstat->stat = SAM_CHECK_COND;
		break;

	case SAS_PROTOCOL_SMP: {
			struct scatterlist *sg_resp = &task->smp_task.smp_resp;
			tstat->stat = SAM_GOOD;
			to = kmap_atomic(sg_page(sg_resp), KM_IRQ0);
			memcpy(to + sg_resp->offset,
				slot->response + sizeof(struct mvs_err_info),
				sg_dma_len(sg_resp));
			kunmap_atomic(to, KM_IRQ0);
			break;
		}

	case SAS_PROTOCOL_SATA:
	case SAS_PROTOCOL_STP:
	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: {
			tstat->stat = mvs_sata_done(mvi, task, slot_idx, 0);
			break;
		}

	default:
		tstat->stat = SAM_CHECK_COND;
		break;
	}

out:
	mvs_slot_task_free(mvi, task, slot, slot_idx);
	if (unlikely(tstat->stat != SAS_QUEUE_FULL))
		mvs_slot_free(mvi, rx_desc);

	spin_unlock(&mvi->lock);
	task->task_done(task);
	spin_lock(&mvi->lock);
	return tstat->stat;
}

static void mvs_release_task(struct mvs_info *mvi, int phy_no)
{
	struct list_head *pos, *n;
	struct mvs_slot_info *slot;
	struct mvs_phy *phy = &mvi->phy[phy_no];
	struct mvs_port *port = phy->port;
	u32 rx_desc;

	if (!port)
		return;

	list_for_each_safe(pos, n, &port->list) {
		slot = container_of(pos, struct mvs_slot_info, list);
		rx_desc = (u32) (slot - mvi->slot_info);
		mvs_slot_complete(mvi, rx_desc, 1);
	}
}

static void mvs_int_full(struct mvs_info *mvi)
{
	void __iomem *regs = mvi->regs;
	u32 tmp, stat;
	int i;

	stat = mr32(INT_STAT);

	mvs_int_rx(mvi, false);

	for (i = 0; i < MVS_MAX_PORTS; i++) {
		tmp = (stat >> i) & (CINT_PORT | CINT_PORT_STOPPED);
		if (tmp)
			mvs_int_port(mvi, i, tmp);
	}

	if (stat & CINT_SRS)
		mvs_int_sata(mvi);

	mw32(INT_STAT, stat);
}

static int mvs_int_rx(struct mvs_info *mvi, bool self_clear)
{
	void __iomem *regs = mvi->regs;
	u32 rx_prod_idx, rx_desc;
	bool attn = false;
	struct pci_dev *pdev = mvi->pdev;

	/* the first dword in the RX ring is special: it contains
	 * a mirror of the hardware's RX producer index, so that
	 * we don't have to stall the CPU reading that register.
	 * The actual RX ring is offset by one dword, due to this.
	 */
	rx_prod_idx = mvi->rx_cons;
	mvi->rx_cons = le32_to_cpu(mvi->rx[0]);
	if (mvi->rx_cons == 0xfff)	/* h/w hasn't touched RX ring yet */
		return 0;

	/* The CMPL_Q may come late, read from register and try again
	* note: if coalescing is enabled,
	* it will need to read from register every time for sure
	*/
	if (mvi->rx_cons == rx_prod_idx)
		mvi->rx_cons = mr32(RX_CONS_IDX) & RX_RING_SZ_MASK;

	if (mvi->rx_cons == rx_prod_idx)
		return 0;

	while (mvi->rx_cons != rx_prod_idx) {

		/* increment our internal RX consumer pointer */
		rx_prod_idx = (rx_prod_idx + 1) & (MVS_RX_RING_SZ - 1);

		rx_desc = le32_to_cpu(mvi->rx[rx_prod_idx + 1]);

		if (likely(rx_desc & RXQ_DONE))
			mvs_slot_complete(mvi, rx_desc, 0);
		if (rx_desc & RXQ_ATTN) {
			attn = true;
			dev_printk(KERN_DEBUG, &pdev->dev, "ATTN %X\n",
				rx_desc);
		} else if (rx_desc & RXQ_ERR) {
			if (!(rx_desc & RXQ_DONE))
				mvs_slot_complete(mvi, rx_desc, 0);
			dev_printk(KERN_DEBUG, &pdev->dev, "RXQ_ERR %X\n",
				rx_desc);
		} else if (rx_desc & RXQ_SLOT_RESET) {
			dev_printk(KERN_DEBUG, &pdev->dev, "Slot reset[%X]\n",
				rx_desc);
			mvs_slot_free(mvi, rx_desc);
		}
	}

	if (attn && self_clear)
		mvs_int_full(mvi);

	return 0;
}

#ifdef MVS_USE_TASKLET
static void mvs_tasklet(unsigned long data)
{
	struct mvs_info *mvi = (struct mvs_info *) data;
	unsigned long flags;

	spin_lock_irqsave(&mvi->lock, flags);

#ifdef MVS_DISABLE_MSI
	mvs_int_full(mvi);
#else
	mvs_int_rx(mvi, true);
#endif
	spin_unlock_irqrestore(&mvi->lock, flags);
}
#endif

static irqreturn_t mvs_interrupt(int irq, void *opaque)
{
	struct mvs_info *mvi = opaque;
	void __iomem *regs = mvi->regs;
	u32 stat;

	stat = mr32(GBL_INT_STAT);

	if (stat == 0 || stat == 0xffffffff)
		return IRQ_NONE;

	/* clear CMD_CMPLT ASAP */
	mw32_f(INT_STAT, CINT_DONE);

#ifndef MVS_USE_TASKLET
	spin_lock(&mvi->lock);

	mvs_int_full(mvi);

	spin_unlock(&mvi->lock);
#else
	tasklet_schedule(&mvi->tasklet);
#endif
	return IRQ_HANDLED;
}

#ifndef MVS_DISABLE_MSI
static irqreturn_t mvs_msi_interrupt(int irq, void *opaque)
{
	struct mvs_info *mvi = opaque;

#ifndef MVS_USE_TASKLET
	spin_lock(&mvi->lock);

	mvs_int_rx(mvi, true);

	spin_unlock(&mvi->lock);
#else
	tasklet_schedule(&mvi->tasklet);
#endif
	return IRQ_HANDLED;
}
#endif

struct mvs_task_exec_info {
	struct sas_task *task;
	struct mvs_cmd_hdr *hdr;
	struct mvs_port *port;
	u32 tag;
	int n_elem;
};

static int mvs_task_prep_smp(struct mvs_info *mvi,
			     struct mvs_task_exec_info *tei)
{
	int elem, rc, i;
	struct sas_task *task = tei->task;
	struct mvs_cmd_hdr *hdr = tei->hdr;
	struct scatterlist *sg_req, *sg_resp;
	u32 req_len, resp_len, tag = tei->tag;
	void *buf_tmp;
	u8 *buf_oaf;
	dma_addr_t buf_tmp_dma;
	struct mvs_prd *buf_prd;
	struct scatterlist *sg;
	struct mvs_slot_info *slot = &mvi->slot_info[tag];
	struct asd_sas_port *sas_port = task->dev->port;
	u32 flags = (tei->n_elem << MCH_PRD_LEN_SHIFT);
{'fraction_non_alphanumeric': 0.1426612543394748, 'fraction_numerical': 0.03887298806168312, 'mean_word_length': 3.575283808006373, 'pattern_counts': {'":': 2, '<': 733, '<?xml version=': 0, '>': 491, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '768385', 'n_tokens_mistral': 45209, 'n_tokens_neox': 41202, 'n_words': 9014}