summaryrefslogtreecommitdiff
path: root/drivers/accel
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2023-10-17 18:07:53 +1000
committerDave Airlie <airlied@redhat.com>2023-10-17 18:07:54 +1000
commit08057253366d916a73e62bafb913d9b659228cc1 (patch)
tree283d73ba79db976064f94453a25c08614a7c111a /drivers/accel
parent614351f41e8c557068f1898eef5bacbca3b20911 (diff)
parent4db74c0fdeb8138f6438d42a015c5dcdb2e6874c (diff)
Merge tag 'drm-habanalabs-next-2023-10-10' of https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux into drm-next
This tag contains habanalabs driver changes for v6.7. The notable changes are: - uAPI changes: - Expose tsc clock sampling to better sync clock information in profiler. - Enhance engine error reporting in the info ioctl. - Block access to the eventfd operations through the control device. - Disable the option of the user to register multiple times with the same offset for timestamp dump by the driver. If a user wants to use the same offset in the timestamp buffer for different interrupt, it needs to first de-register the offset. - When exporting dma-buf (for p2p), force the user to specify size/offset in multiples of PAGE_SIZE. This is instead of the driver doing the rounding to PAGE_SIZE, which has caused the driver to map more memory than was intended by the user. - New features and improvements: - Complete the move of the driver to the accel subsystem by removing the custom habanalabs class and major and registering to accel subsystem. - Move the firmware interface files to include/linux/habanalabs. This is a pre-requisite for upstreaming the NIC drivers of Gaudi (as they need to include those files). - Perform device hard-reset upon PCIe AXI drain event to prevent the failure from cascading to different IP blocks in the SoC. In secured environments, this is done automatically by the firmware. - Print device name when it is removed for better debuggability. - Add support for trace of dma map sgtable operations. - Optimize handling of user interrupts by splitting the interrupts to two lists. One list for fast handling and second list for handling with timestamp recording, which is slower. - Prevent double device hard-reset due to 2 adjacent H/W events. - Set device status 'malfunction' while in rmmod. - Firmware related fixes: - Extend preboot timeout because preboot loading might take longer than expected in certain cases. - Add a protection mechanism for the Event Queue. In case it is full, the firmware will be able to notify about it through a dedicated interrupt. - Perform device hard-reset in case scrubbing of memory has failed. - Bug fixes and code cleanups: - Small fixes of dma-buf handling in Gaudi2, such as handling an offset != 0, using the correct exported size, creation of sg table. - Fix spmu mask creation. - Fix bug in wait for cs completion for decoder workloads. - Cleanup Greco name from documentation. - Fix bug in recording timestamp during cs completion interrupt handling. - Fix CoreSight ETF configuration and flush logic. - Fix small bug in hpriv_list handling (the list that contains the private data per process that opens our device). Signed-off-by: Dave Airlie <airlied@redhat.com> # -----BEGIN PGP SIGNATURE----- # # iQEzBAABCgAdFiEE7TEboABC71LctBLFZR1NuKta54AFAmUlHoQACgkQZR1NuKta # 54DsXQf8CW+W4iWJf5UDTj/E/giu9rVRrsUsU0hhCcXbecIxRsLObYXtulENu5/u # VuEAo/tAvo0LUKi8pdIv6ernDKaxZ1+fimlfXMCzllAA/ts3yp1NgunprsIsx3tv # YgcJ2GNR8UlVZ1qYuZl+4dOTyD0yfRMROUXBe7wqKnUXOEepOiLBxq6W15tZiJnx # L+V0yGkNk6pAoADIXLW9EgEXiN/bJZCXGPWp06i/Nz7cHIHJGoV59wAqftqllCtk # 8ZMkLByjlQKPhc5AgWBtKE8EGVip3sm7b/Q2Gq0ZXdZiebyVJ+AjuuDOdtq1UCIw # Rcp2576E7rByIBu3RAFlrioWhuR5Zw== # =2ien # -----END PGP SIGNATURE----- # gpg: Signature made Tue 10 Oct 2023 19:51:00 AEST # gpg: using RSA key ED311BA00042EF52DCB412C5651D4DB8AB5AE780 # gpg: Can't check signature: No public key From: Oded Gabbay <ogabbay@kernel.org> Link: https://patchwork.freedesktop.org/patch/msgid/ZSUfiX4J7v4Wn0cU@ogabbay-vm-u22.habana-labs.com
Diffstat (limited to 'drivers/accel')
-rw-r--r--drivers/accel/drm_accel.c21
-rw-r--r--drivers/accel/habanalabs/common/command_buffer.c5
-rw-r--r--drivers/accel/habanalabs/common/command_submission.c488
-rw-r--r--drivers/accel/habanalabs/common/context.c9
-rw-r--r--drivers/accel/habanalabs/common/debugfs.c22
-rw-r--r--drivers/accel/habanalabs/common/device.c425
-rw-r--r--drivers/accel/habanalabs/common/firmware_if.c45
-rw-r--r--drivers/accel/habanalabs/common/habanalabs.h212
-rw-r--r--drivers/accel/habanalabs/common/habanalabs_drv.c186
-rw-r--r--drivers/accel/habanalabs/common/habanalabs_ioctl.c112
-rw-r--r--drivers/accel/habanalabs/common/irq.c180
-rw-r--r--drivers/accel/habanalabs/common/memory.c308
-rw-r--r--drivers/accel/habanalabs/gaudi/gaudi.c17
-rw-r--r--drivers/accel/habanalabs/gaudi/gaudiP.h2
-rw-r--r--drivers/accel/habanalabs/gaudi/gaudi_coresight.c12
-rw-r--r--drivers/accel/habanalabs/gaudi2/gaudi2.c487
-rw-r--r--drivers/accel/habanalabs/gaudi2/gaudi2P.h4
-rw-r--r--drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c46
-rw-r--r--drivers/accel/habanalabs/gaudi2/gaudi2_security.c21
-rw-r--r--drivers/accel/habanalabs/goya/goya.c10
-rw-r--r--drivers/accel/habanalabs/goya/goyaP.h2
-rw-r--r--drivers/accel/habanalabs/goya/goya_coresight.c10
-rw-r--r--drivers/accel/habanalabs/include/common/cpucp_if.h1401
-rw-r--r--drivers/accel/habanalabs/include/common/hl_boot_if.h785
-rw-r--r--drivers/accel/habanalabs/include/gaudi/gaudi_fw_if.h32
-rw-r--r--drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h7
-rw-r--r--drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h16
27 files changed, 1726 insertions, 3139 deletions
diff --git a/drivers/accel/drm_accel.c b/drivers/accel/drm_accel.c
index 94b4ac12cf24..294b572a9c33 100644
--- a/drivers/accel/drm_accel.c
+++ b/drivers/accel/drm_accel.c
@@ -21,7 +21,6 @@ static DEFINE_SPINLOCK(accel_minor_lock);
static struct idr accel_minors_idr;
static struct dentry *accel_debugfs_root;
-static struct class *accel_class;
static struct device_type accel_sysfs_device_minor = {
.name = "accel_minor"
@@ -32,23 +31,19 @@ static char *accel_devnode(const struct device *dev, umode_t *mode)
return kasprintf(GFP_KERNEL, "accel/%s", dev_name(dev));
}
+static const struct class accel_class = {
+ .name = "accel",
+ .devnode = accel_devnode,
+};
+
static int accel_sysfs_init(void)
{
- accel_class = class_create("accel");
- if (IS_ERR(accel_class))
- return PTR_ERR(accel_class);
-
- accel_class->devnode = accel_devnode;
-
- return 0;
+ return class_register(&accel_class);
}
static void accel_sysfs_destroy(void)
{
- if (IS_ERR_OR_NULL(accel_class))
- return;
- class_destroy(accel_class);
- accel_class = NULL;
+ class_unregister(&accel_class);
}
static int accel_name_info(struct seq_file *m, void *data)
@@ -117,7 +112,7 @@ void accel_debugfs_register(struct drm_device *dev)
void accel_set_device_instance_params(struct device *kdev, int index)
{
kdev->devt = MKDEV(ACCEL_MAJOR, index);
- kdev->class = accel_class;
+ kdev->class = &accel_class;
kdev->type = &accel_sysfs_device_minor;
}
diff --git a/drivers/accel/habanalabs/common/command_buffer.c b/drivers/accel/habanalabs/common/command_buffer.c
index 08f7aee42624..0f0d295116e7 100644
--- a/drivers/accel/habanalabs/common/command_buffer.c
+++ b/drivers/accel/habanalabs/common/command_buffer.c
@@ -361,10 +361,11 @@ out:
return rc;
}
-int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
- union hl_cb_args *args = data;
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
struct hl_device *hdev = hpriv->hdev;
+ union hl_cb_args *args = data;
u64 handle = 0, device_va = 0;
enum hl_device_status status;
u32 usage_cnt = 0;
diff --git a/drivers/accel/habanalabs/common/command_submission.c b/drivers/accel/habanalabs/common/command_submission.c
index c23829dab97a..3aa6eeef443b 100644
--- a/drivers/accel/habanalabs/common/command_submission.c
+++ b/drivers/accel/habanalabs/common/command_submission.c
@@ -31,6 +31,24 @@ enum hl_cs_wait_status {
CS_WAIT_STATUS_GONE
};
+/*
+ * Data used while handling wait/timestamp nodes.
+ * The purpose of this struct is to store the needed data for both operations
+ * in one variable instead of passing large number of arguments to functions.
+ */
+struct wait_interrupt_data {
+ struct hl_user_interrupt *interrupt;
+ struct hl_mmap_mem_buf *buf;
+ struct hl_mem_mgr *mmg;
+ struct hl_cb *cq_cb;
+ u64 ts_handle;
+ u64 ts_offset;
+ u64 cq_handle;
+ u64 cq_offset;
+ u64 target_value;
+ u64 intr_timeout_us;
+};
+
static void job_wq_completion(struct work_struct *work);
static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, u64 timeout_us, u64 seq,
enum hl_cs_wait_status *status, s64 *timestamp);
@@ -1079,19 +1097,22 @@ static void
wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)
{
struct hl_user_pending_interrupt *pend, *temp;
+ unsigned long flags;
- spin_lock(&interrupt->wait_list_lock);
- list_for_each_entry_safe(pend, temp, &interrupt->wait_list_head, wait_list_node) {
- if (pend->ts_reg_info.buf) {
- list_del(&pend->wait_list_node);
- hl_mmap_mem_buf_put(pend->ts_reg_info.buf);
- hl_cb_put(pend->ts_reg_info.cq_cb);
- } else {
- pend->fence.error = -EIO;
- complete_all(&pend->fence.completion);
- }
+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+ list_for_each_entry_safe(pend, temp, &interrupt->wait_list_head, list_node) {
+ pend->fence.error = -EIO;
+ complete_all(&pend->fence.completion);
}
- spin_unlock(&interrupt->wait_list_lock);
+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
+
+ spin_lock_irqsave(&interrupt->ts_list_lock, flags);
+ list_for_each_entry_safe(pend, temp, &interrupt->ts_list_head, list_node) {
+ list_del(&pend->list_node);
+ hl_mmap_mem_buf_put(pend->ts_reg_info.buf);
+ hl_cb_put(pend->ts_reg_info.cq_cb);
+ }
+ spin_unlock_irqrestore(&interrupt->ts_list_lock, flags);
}
void hl_release_pending_user_interrupts(struct hl_device *hdev)
@@ -1730,16 +1751,11 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,
/* Need to wait for restore completion before execution phase */
if (num_chunks) {
enum hl_cs_wait_status status;
-wait_again:
+
ret = _hl_cs_wait_ioctl(hdev, ctx,
jiffies_to_usecs(hdev->timeout_jiffies),
*cs_seq, &status, NULL);
if (ret) {
- if (ret == -ERESTARTSYS) {
- usleep_range(100, 200);
- goto wait_again;
- }
-
dev_err(hdev->dev,
"Restore CS for context %d failed to complete %d\n",
ctx->asid, ret);
@@ -2539,8 +2555,9 @@ static int cs_ioctl_flush_pci_hbw_writes(struct hl_fpriv *hpriv)
return 0;
}
-int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_cs_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
union hl_cs_args *args = data;
enum hl_cs_type cs_type = 0;
u64 cs_seq = ULONG_MAX;
@@ -3197,166 +3214,241 @@ static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data)
return 0;
}
-static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,
- struct hl_cb *cq_cb,
- u64 ts_offset, u64 cq_offset, u64 target_value,
- spinlock_t *wait_list_lock,
- struct hl_user_pending_interrupt **pend)
+static inline void set_record_cq_info(struct hl_user_pending_interrupt *record,
+ struct hl_cb *cq_cb, u32 cq_offset, u32 target_value)
{
- struct hl_ts_buff *ts_buff = buf->private;
- struct hl_user_pending_interrupt *requested_offset_record =
- (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
- ts_offset;
- struct hl_user_pending_interrupt *cb_last =
- (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
+ record->ts_reg_info.cq_cb = cq_cb;
+ record->cq_kernel_addr = (u64 *) cq_cb->kernel_address + cq_offset;
+ record->cq_target_value = target_value;
+}
+
+static int validate_and_get_ts_record(struct device *dev,
+ struct hl_ts_buff *ts_buff, u64 ts_offset,
+ struct hl_user_pending_interrupt **req_event_record)
+{
+ struct hl_user_pending_interrupt *ts_cb_last;
+
+ *req_event_record = (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
+ ts_offset;
+ ts_cb_last = (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
(ts_buff->kernel_buff_size / sizeof(struct hl_user_pending_interrupt));
- unsigned long iter_counter = 0;
- u64 current_cq_counter;
- ktime_t timestamp;
/* Validate ts_offset not exceeding last max */
- if (requested_offset_record >= cb_last) {
- dev_err(buf->mmg->dev, "Ts offset exceeds max CB offset(0x%llx)\n",
- (u64)(uintptr_t)cb_last);
+ if (*req_event_record >= ts_cb_last) {
+ dev_err(dev, "Ts offset(%llu) exceeds max CB offset(0x%llx)\n",
+ ts_offset, (u64)(uintptr_t)ts_cb_last);
return -EINVAL;
}
- timestamp = ktime_get();
+ return 0;
+}
-start_over:
- spin_lock(wait_list_lock);
+static void unregister_timestamp_node(struct hl_device *hdev,
+ struct hl_user_pending_interrupt *record, bool need_lock)
+{
+ struct hl_user_interrupt *interrupt = record->ts_reg_info.interrupt;
+ bool ts_rec_found = false;
+ unsigned long flags;
- /* Unregister only if we didn't reach the target value
- * since in this case there will be no handling in irq context
- * and then it's safe to delete the node out of the interrupt list
- * then re-use it on other interrupt
- */
- if (requested_offset_record->ts_reg_info.in_use) {
- current_cq_counter = *requested_offset_record->cq_kernel_addr;
- if (current_cq_counter < requested_offset_record->cq_target_value) {
- list_del(&requested_offset_record->wait_list_node);
- spin_unlock(wait_list_lock);
+ if (need_lock)
+ spin_lock_irqsave(&interrupt->ts_list_lock, flags);
- hl_mmap_mem_buf_put(requested_offset_record->ts_reg_info.buf);
- hl_cb_put(requested_offset_record->ts_reg_info.cq_cb);
+ if (record->ts_reg_info.in_use) {
+ record->ts_reg_info.in_use = false;
+ list_del(&record->list_node);
+ ts_rec_found = true;
+ }
- dev_dbg(buf->mmg->dev,
- "ts node removed from interrupt list now can re-use\n");
- } else {
- dev_dbg(buf->mmg->dev,
- "ts node in middle of irq handling\n");
-
- /* irq thread handling in the middle give it time to finish */
- spin_unlock(wait_list_lock);
- usleep_range(100, 1000);
- if (++iter_counter == MAX_TS_ITER_NUM) {
- dev_err(buf->mmg->dev,
- "Timestamp offset processing reached timeout of %lld ms\n",
- ktime_ms_delta(ktime_get(), timestamp));
- return -EAGAIN;
- }
+ if (need_lock)
+ spin_unlock_irqrestore(&interrupt->ts_list_lock, flags);
- goto start_over;
+ /* Put refcounts that were taken when we registered the event */
+ if (ts_rec_found) {
+ hl_mmap_mem_buf_put(record->ts_reg_info.buf);
+ hl_cb_put(record->ts_reg_info.cq_cb);
+ }
+}
+
+static int ts_get_and_handle_kernel_record(struct hl_device *hdev, struct hl_ctx *ctx,
+ struct wait_interrupt_data *data, unsigned long *flags,
+ struct hl_user_pending_interrupt **pend)
+{
+ struct hl_user_pending_interrupt *req_offset_record;
+ struct hl_ts_buff *ts_buff = data->buf->private;
+ bool need_lock = false;
+ int rc;
+
+ rc = validate_and_get_ts_record(data->buf->mmg->dev, ts_buff, data->ts_offset,
+ &req_offset_record);
+ if (rc)
+ return rc;
+
+ /* In case the node already registered, need to unregister first then re-use */
+ if (req_offset_record->ts_reg_info.in_use) {
+ dev_dbg(data->buf->mmg->dev,
+ "Requested record %p is in use on irq: %u ts addr: %p, unregister first then put on irq: %u\n",
+ req_offset_record,
+ req_offset_record->ts_reg_info.interrupt->interrupt_id,
+ req_offset_record->ts_reg_info.timestamp_kernel_addr,
+ data->interrupt->interrupt_id);
+ /*
+ * Since interrupt here can be different than the one the node currently registered
+ * on, and we don't want to lock two lists while we're doing unregister, so
+ * unlock the new interrupt wait list here and acquire the lock again after you done
+ */
+ if (data->interrupt->interrupt_id !=
+ req_offset_record->ts_reg_info.interrupt->interrupt_id) {
+
+ need_lock = true;
+ spin_unlock_irqrestore(&data->interrupt->ts_list_lock, *flags);
}
- } else {
- /* Fill up the new registration node info */
- requested_offset_record->ts_reg_info.buf = buf;
- requested_offset_record->ts_reg_info.cq_cb = cq_cb;
- requested_offset_record->ts_reg_info.timestamp_kernel_addr =
- (u64 *) ts_buff->user_buff_address + ts_offset;
- requested_offset_record->cq_kernel_addr =
- (u64 *) cq_cb->kernel_address + cq_offset;
- requested_offset_record->cq_target_value = target_value;
- spin_unlock(wait_list_lock);
+ unregister_timestamp_node(hdev, req_offset_record, need_lock);
+
+ if (need_lock)
+ spin_lock_irqsave(&data->interrupt->ts_list_lock, *flags);
}
- *pend = requested_offset_record;
+ /* Fill up the new registration node info and add it to the list */
+ req_offset_record->ts_reg_info.in_use = true;
+ req_offset_record->ts_reg_info.buf = data->buf;
+ req_offset_record->ts_reg_info.timestamp_kernel_addr =
+ (u64 *) ts_buff->user_buff_address + data->ts_offset;
+ req_offset_record->ts_reg_info.interrupt = data->interrupt;
+ set_record_cq_info(req_offset_record, data->cq_cb, data->cq_offset,
+ data->target_value);
- dev_dbg(buf->mmg->dev, "Found available node in TS kernel CB %p\n",
- requested_offset_record);
- return 0;
+ *pend = req_offset_record;
+
+ return rc;
+}
+
+static int _hl_interrupt_ts_reg_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
+ struct wait_interrupt_data *data,
+ u32 *status, u64 *timestamp)
+{
+ struct hl_user_pending_interrupt *pend;
+ unsigned long flags;
+ int rc = 0;
+
+ hl_ctx_get(ctx);
+
+ data->cq_cb = hl_cb_get(data->mmg, data->cq_handle);
+ if (!data->cq_cb) {
+ rc = -EINVAL;
+ goto put_ctx;
+ }
+
+ /* Validate the cq offset */
+ if (((u64 *) data->cq_cb->kernel_address + data->cq_offset) >=
+ ((u64 *) data->cq_cb->kernel_address + (data->cq_cb->size / sizeof(u64)))) {
+ rc = -EINVAL;
+ goto put_cq_cb;
+ }
+
+ dev_dbg(hdev->dev, "Timestamp registration: interrupt id: %u, handle: 0x%llx, ts offset: %llu, cq_offset: %llu\n",
+ data->interrupt->interrupt_id, data->ts_handle,
+ data->ts_offset, data->cq_offset);
+
+ data->buf = hl_mmap_mem_buf_get(data->mmg, data->ts_handle);
+ if (!data->buf) {
+ rc = -EINVAL;
+ goto put_cq_cb;
+ }
+
+ spin_lock_irqsave(&data->interrupt->ts_list_lock, flags);
+
+ /* get ts buffer record */
+ rc = ts_get_and_handle_kernel_record(hdev, ctx, data, &flags, &pend);
+ if (rc) {
+ spin_unlock_irqrestore(&data->interrupt->ts_list_lock, flags);
+ goto put_ts_buff;
+ }
+
+ /* We check for completion value as interrupt could have been received
+ * before we add the timestamp node to the ts list.
+ */
+ if (*pend->cq_kernel_addr >= data->target_value) {
+ spin_unlock_irqrestore(&data->interrupt->ts_list_lock, flags);
+
+ dev_dbg(hdev->dev, "Target value already reached release ts record: pend: %p, offset: %llu, interrupt: %u\n",
+ pend, data->ts_offset, data->interrupt->interrupt_id);
+
+ pend->ts_reg_info.in_use = 0;
+ *status = HL_WAIT_CS_STATUS_COMPLETED;
+ *pend->ts_reg_info.timestamp_kernel_addr = ktime_get_ns();
+
+ goto put_ts_buff;
+ }
+
+ list_add_tail(&pend->list_node, &data->interrupt->ts_list_head);
+ spin_unlock_irqrestore(&data->interrupt->ts_list_lock, flags);
+
+ rc = *status = HL_WAIT_CS_STATUS_COMPLETED;
+
+ hl_ctx_put(ctx);
+
+ return rc;
+
+put_ts_buff:
+ hl_mmap_mem_buf_put(data->buf);
+put_cq_cb:
+ hl_cb_put(data->cq_cb);
+put_ctx:
+ hl_ctx_put(ctx);
+
+ return rc;
}
static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
- struct hl_mem_mgr *cb_mmg, struct hl_mem_mgr *mmg,
- u64 timeout_us, u64 cq_counters_handle, u64 cq_counters_offset,
- u64 target_value, struct hl_user_interrupt *interrupt,
- bool register_ts_record, u64 ts_handle, u64 ts_offset,
+ struct wait_interrupt_data *data,
u32 *status, u64 *timestamp)
{
struct hl_user_pending_interrupt *pend;
- struct hl_mmap_mem_buf *buf;
- struct hl_cb *cq_cb;
- unsigned long timeout;
+ unsigned long timeout, flags;
long completion_rc;
int rc = 0;
- timeout = hl_usecs64_to_jiffies(timeout_us);
+ timeout = hl_usecs64_to_jiffies(data->intr_timeout_us);
hl_ctx_get(ctx);
- cq_cb = hl_cb_get(cb_mmg, cq_counters_handle);
- if (!cq_cb) {
+ data->cq_cb = hl_cb_get(data->mmg, data->cq_handle);
+ if (!data->cq_cb) {
rc = -EINVAL;
goto put_ctx;
}
/* Validate the cq offset */
- if (((u64 *) cq_cb->kernel_address + cq_counters_offset) >=
- ((u64 *) cq_cb->kernel_address + (cq_cb->size / sizeof(u64)))) {
+ if (((u64 *) data->cq_cb->kernel_address + data->cq_offset) >=
+ ((u64 *) data->cq_cb->kernel_address + (data->cq_cb->size / sizeof(u64)))) {
rc = -EINVAL;
goto put_cq_cb;
}
- if (register_ts_record) {
- dev_dbg(hdev->dev, "Timestamp registration: interrupt id: %u, ts offset: %llu, cq_offset: %llu\n",
- interrupt->interrupt_id, ts_offset, cq_counters_offset);
- buf = hl_mmap_mem_buf_get(mmg, ts_handle);
- if (!buf) {
- rc = -EINVAL;
- goto put_cq_cb;
- }
-
- /* get ts buffer record */
- rc = ts_buff_get_kernel_ts_record(buf, cq_cb, ts_offset,
- cq_counters_offset, target_value,
- &interrupt->wait_list_lock, &pend);
- if (rc)
- goto put_ts_buff;
- } else {
- pend = kzalloc(sizeof(*pend), GFP_KERNEL);
- if (!pend) {
- rc = -ENOMEM;
- goto put_cq_cb;
- }
- hl_fence_init(&pend->fence, ULONG_MAX);
- pend->cq_kernel_addr = (u64 *) cq_cb->kernel_address + cq_counters_offset;
- pend->cq_target_value = target_value;
+ pend = kzalloc(sizeof(*pend), GFP_KERNEL);
+ if (!pend) {
+ rc = -ENOMEM;
+ goto put_cq_cb;
}
- spin_lock(&interrupt->wait_list_lock);
+ hl_fence_init(&pend->fence, ULONG_MAX);
+ pend->cq_kernel_addr = (u64 *) data->cq_cb->kernel_address + data->cq_offset;
+ pend->cq_target_value = data->target_value;
+ spin_lock_irqsave(&data->interrupt->wait_list_lock, flags);
+
/* We check for completion value as interrupt could have been received
- * before we added the node to the wait list
+ * before we add the wait node to the wait list.
*/
- if (*pend->cq_kernel_addr >= target_value) {
- if (register_ts_record)
- pend->ts_reg_info.in_use = 0;
- spin_unlock(&interrupt->wait_list_lock);
+ if (*pend->cq_kernel_addr >= data->target_value || (!data->intr_timeout_us)) {
+ spin_unlock_irqrestore(&data->interrupt->wait_list_lock, flags);
- *status = HL_WAIT_CS_STATUS_COMPLETED;
+ if (*pend->cq_kernel_addr >= data->target_value)
+ *status = HL_WAIT_CS_STATUS_COMPLETED;
+ else
+ *status = HL_WAIT_CS_STATUS_BUSY;
- if (register_ts_record) {
- *pend->ts_reg_info.timestamp_kernel_addr = ktime_get_ns();
- goto put_ts_buff;
- } else {
- pend->fence.timestamp = ktime_get();
- goto set_timestamp;
- }
- } else if (!timeout_us) {
- spin_unlock(&interrupt->wait_list_lock);
- *status = HL_WAIT_CS_STATUS_BUSY;
pend->fence.timestamp = ktime_get();
goto set_timestamp;
}
@@ -3366,55 +3458,38 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
* Note that we cannot have sorted list by target value,
* in order to shorten the list pass loop, since
* same list could have nodes for different cq counter handle.
- * Note:
- * Mark ts buff offset as in use here in the spinlock protection area
- * to avoid getting in the re-use section in ts_buff_get_kernel_ts_record
- * before adding the node to the list. this scenario might happen when
- * multiple threads are racing on same offset and one thread could
- * set the ts buff in ts_buff_get_kernel_ts_record then the other thread
- * takes over and get to ts_buff_get_kernel_ts_record and then we will try
- * to re-use the same ts buff offset, and will try to delete a non existing
- * node from the list.
*/
- if (register_ts_record)
- pend->ts_reg_info.in_use = 1;
-
- list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
- spin_unlock(&interrupt->wait_list_lock);
-
- if (register_ts_record) {
- rc = *status = HL_WAIT_CS_STATUS_COMPLETED;
- goto ts_registration_exit;
- }
+ list_add_tail(&pend->list_node, &data->interrupt->wait_list_head);
+ spin_unlock_irqrestore(&data->interrupt->wait_list_lock, flags);
/* Wait for interrupt handler to signal completion */
completion_rc = wait_for_completion_interruptible_timeout(&pend->fence.completion,
timeout);
if (completion_rc > 0) {
- *status = HL_WAIT_CS_STATUS_COMPLETED;
+ if (pend->fence.error == -EIO) {
+ dev_err_ratelimited(hdev->dev,
+ "interrupt based wait ioctl aborted(error:%d) due to a reset cycle initiated\n",
+ pend->fence.error);
+ rc = -EIO;
+ *status = HL_WAIT_CS_STATUS_ABORTED;
+ } else {
+ *status = HL_WAIT_CS_STATUS_COMPLETED;
+ }
} else {
if (completion_rc == -ERESTARTSYS) {
dev_err_ratelimited(hdev->dev,
"user process got signal while waiting for interrupt ID %d\n",
- interrupt->interrupt_id);
+ data->interrupt->interrupt_id);
rc = -EINTR;
*status = HL_WAIT_CS_STATUS_ABORTED;
} else {
- if (pend->fence.error == -EIO) {
- dev_err_ratelimited(hdev->dev,
- "interrupt based wait ioctl aborted(error:%d) due to a reset cycle initiated\n",
- pend->fence.error);
- rc = -EIO;
- *status = HL_WAIT_CS_STATUS_ABORTED;
- } else {
- /* The wait has timed-out. We don't know anything beyond that
- * because the workload wasn't submitted through the driver.
- * Therefore, from driver's perspective, the workload is still
- * executing.
- */
- rc = 0;
- *status = HL_WAIT_CS_STATUS_BUSY;
- }
+ /* The wait has timed-out. We don't know anything beyond that
+ * because the workload was not submitted through the driver.
+ * Therefore, from driver's perspective, the workload is still
+ * executing.
+ */
+ rc = 0;
+ *status = HL_WAIT_CS_STATUS_BUSY;
}
}
@@ -3424,23 +3499,20 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx,
* for ts record, the node will be deleted in the irq handler after
* we reach the target value.
*/
- spin_lock(&interrupt->wait_list_lock);
- list_del(&pend->wait_list_node);
- spin_unlock(&interrupt->wait_list_lock);
+ spin_lock_irqsave(&data->interrupt->wait_list_lock, flags);
+ list_del(&pend->list_node);
+ spin_unlock_irqrestore(&data->interrupt->wait_list_lock, flags);
set_timestamp:
*timestamp = ktime_to_ns(pend->fence.timestamp);
kfree(pend);
- hl_cb_put(cq_cb);
-ts_registration_exit:
+ hl_cb_put(data->cq_cb);
hl_ctx_put(ctx);
return rc;
-put_ts_buff:
- hl_mmap_mem_buf_put(buf);
put_cq_cb:
- hl_cb_put(cq_cb);
+ hl_cb_put(data->cq_cb);
put_ctx:
hl_ctx_put(ctx);
@@ -3454,7 +3526,7 @@ static int _hl_interrupt_wait_ioctl_user_addr(struct hl_device *hdev, struct hl_
u64 *timestamp)
{
struct hl_user_pending_interrupt *pend;
- unsigned long timeout;
+ unsigned long timeout, flags;
u64 completion_value;
long completion_rc;
int rc = 0;
@@ -3474,9 +3546,9 @@ static int _hl_interrupt_wait_ioctl_user_addr(struct hl_device *hdev, struct hl_
/* Add pending user interrupt to relevant list for the interrupt
* handler to monitor
*/
- spin_lock(&interrupt->wait_list_lock);
- list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head);
- spin_unlock(&interrupt->wait_list_lock);
+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+ list_add_tail(&pend->list_node, &interrupt->wait_list_head);
+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
/* We check for completion value as interrupt could have been received
* before we added the node to the wait list
@@ -3507,14 +3579,14 @@ wait_again:
* If comparison fails, keep waiting until timeout expires
*/
if (completion_rc > 0) {
- spin_lock(&interrupt->wait_list_lock);
+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
/* reinit_completion must be called before we check for user
* completion value, otherwise, if interrupt is received after
* the comparison and before the next wait_for_completion,
* we will reach timeout and fail
*/
reinit_completion(&pend->fence.completion);
- spin_unlock(&interrupt->wait_list_lock);
+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
if (copy_from_user(&completion_value, u64_to_user_ptr(user_address), 8)) {
dev_err(hdev->dev, "Failed to copy completion value from user\n");
@@ -3551,9 +3623,9 @@ wait_again:
}
remove_pending_user_interrupt:
- spin_lock(&interrupt->wait_list_lock);
- list_del(&pend->wait_list_node);
- spin_unlock(&interrupt->wait_list_lock);
+ spin_lock_irqsave(&interrupt->wait_list_lock, flags);
+ list_del(&pend->list_node);
+ spin_unlock_irqrestore(&interrupt->wait_list_lock, flags);
*timestamp = ktime_to_ns(pend->fence.timestamp);
@@ -3611,19 +3683,42 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
return -EINVAL;
}
- if (args->in.flags & HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ)
- rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->mem_mgr, &hpriv->mem_mgr,
- args->in.interrupt_timeout_us, args->in.cq_counters_handle,
- args->in.cq_counters_offset,
- args->in.target, interrupt,
- !!(args->in.flags & HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT),
- args->in.timestamp_handle, args->in.timestamp_offset,
- &status, &timestamp);
- else
+ if (args->in.flags & HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ) {
+ struct wait_interrupt_data wait_intr_data = {0};
+
+ wait_intr_data.interrupt = interrupt;
+ wait_intr_data.mmg = &hpriv->mem_mgr;
+ wait_intr_data.cq_handle = args->in.cq_counters_handle;
+ wait_intr_data.cq_offset = args->in.cq_counters_offset;
+ wait_intr_data.ts_handle = args->in.timestamp_handle;
+ wait_intr_data.ts_offset = args->in.timestamp_offset;
+ wait_intr_data.target_value = args->in.target;
+ wait_intr_data.intr_timeout_us = args->in.interrupt_timeout_us;
+
+ if (args->in.flags & HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT) {
+ /*
+ * Allow only one registration at a time. this is needed in order to prevent
+ * issues while handling the flow of re-use of the same offset.
+ * Since the registration flow is protected only by the interrupt lock,
+ * re-use flow might request to move ts node to another interrupt list,
+ * and in such case we're not protected.
+ */
+ mutex_lock(&hpriv->ctx->ts_reg_lock);
+
+ rc = _hl_interrupt_ts_reg_ioctl(hdev, hpriv->ctx, &wait_intr_data,
+ &status, &timestamp);
+
+ mutex_unlock(&hpriv->ctx->ts_reg_lock);
+ } else
+ rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &wait_intr_data,
+ &status, &timestamp);
+ } else {
rc = _hl_interrupt_wait_ioctl_user_addr(hdev, hpriv->ctx,
args->in.interrupt_timeout_us, args->in.addr,
args->in.target, interrupt, &status,
&timestamp);
+ }
+
if (rc)
return rc;
@@ -3638,8 +3733,9 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
return 0;
}
-int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_wait_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
struct hl_device *hdev = hpriv->hdev;
union hl_wait_cs_args *args = data;
u32 flags = args->in.flags;
diff --git a/drivers/accel/habanalabs/common/context.c b/drivers/accel/habanalabs/common/context.c
index 9c8b1b37b510..b83141f58319 100644
--- a/drivers/accel/habanalabs/common/context.c
+++ b/drivers/accel/habanalabs/common/context.c
@@ -102,7 +102,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
kfree(ctx->cs_pending);
if (ctx->asid != HL_KERNEL_ASID_ID) {
- dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);
+ dev_dbg(hdev->dev, "closing user context, asid=%u\n", ctx->asid);
/* The engines are stopped as there is no executing CS, but the
* Coresight might be still working by accessing addresses
@@ -119,6 +119,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
hl_vm_ctx_fini(ctx);
hl_asid_free(hdev, ctx->asid);
hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
+ mutex_destroy(&ctx->ts_reg_lock);
} else {
dev_dbg(hdev->dev, "closing kernel context\n");
hdev->asic_funcs->ctx_fini(ctx);
@@ -198,6 +199,7 @@ out_err:
int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
{
+ char task_comm[TASK_COMM_LEN];
int rc = 0, i;
ctx->hdev = hdev;
@@ -267,7 +269,10 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
hl_encaps_sig_mgr_init(&ctx->sig_mgr);
- dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
+ mutex_init(&ctx->ts_reg_lock);
+
+ dev_dbg(hdev->dev, "create user context, comm=\"%s\", asid=%u\n",
+ get_task_comm(task_comm, current), ctx->asid);
}
return 0;
diff --git a/drivers/accel/habanalabs/common/debugfs.c b/drivers/accel/habanalabs/common/debugfs.c
index 9e84a47a21dc..01f071d52570 100644
--- a/drivers/accel/habanalabs/common/debugfs.c
+++ b/drivers/accel/habanalabs/common/debugfs.c
@@ -18,8 +18,6 @@
#define MMU_KBUF_SIZE (MMU_ADDR_BUF_SIZE + MMU_ASID_BUF_SIZE)
#define I2C_MAX_TRANSACTION_LEN 8
-static struct dentry *hl_debug_root;
-
static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,
u8 i2c_reg, u8 i2c_len, u64 *val)
{
@@ -1788,20 +1786,14 @@ void hl_debugfs_add_device(struct hl_device *hdev)
{
struct hl_dbg_device_entry *dev_entry = &hdev->hl_debugfs;
- dev_entry->root = debugfs_create_dir(dev_name(hdev->dev), hl_debug_root);
+ dev_entry->root = hdev->drm.accel->debugfs_root;
add_files_to_device(hdev, dev_entry, dev_entry->root);
+
if (!hdev->asic_prop.fw_security_enabled)
add_secured_nodes(dev_entry, dev_entry->root);
}
-void hl_debugfs_remove_device(struct hl_device *hdev)
-{
- struct hl_dbg_device_entry *entry = &hdev->hl_debugfs;
-
- debugfs_remove_recursive(entry->root);
-}
-
void hl_debugfs_add_file(struct hl_fpriv *hpriv)
{
struct hl_dbg_device_entry *dev_entry = &hpriv->hdev->hl_debugfs;
@@ -1932,13 +1924,3 @@ void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
up_write(&dev_entry->state_dump_sem);
}
-
-void __init hl_debugfs_init(void)
-{
- hl_debug_root = debugfs_create_dir("habanalabs", NULL);
-}
-
-void hl_debugfs_fini(void)
-{
- debugfs_remove_recursive(hl_debug_root);
-}
diff --git a/drivers/accel/habanalabs/common/device.c b/drivers/accel/habanalabs/common/device.c
index b97339d1f7c6..9711e8fc979d 100644
--- a/drivers/accel/habanalabs/common/device.c
+++ b/drivers/accel/habanalabs/common/device.c
@@ -14,11 +14,14 @@
#include <linux/hwmon.h>
#include <linux/vmalloc.h>
+#include <drm/drm_accel.h>
+#include <drm/drm_drv.h>
+
#include <trace/events/habanalabs.h>
#define HL_RESET_DELAY_USEC 10000 /* 10ms */
-#define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC 5
+#define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC 30
enum dma_alloc_type {
DMA_ALLOC_COHERENT,
@@ -185,7 +188,36 @@ void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *
hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
}
-int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
+int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir, const char *caller)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct scatterlist *sg;
+ int rc, i;
+
+ rc = hdev->asic_funcs->dma_map_sgtable(hdev, sgt, dir);
+ if (rc)
+ return rc;
+
+ if (!trace_habanalabs_dma_map_page_enabled())
+ return 0;
+
+ for_each_sgtable_dma_sg(sgt, sg, i)
+ trace_habanalabs_dma_map_page(hdev->dev,
+ page_to_phys(sg_page(sg)),
+ sg->dma_address - prop->device_dma_offset_for_host_access,
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->dma_length,
+#else
+ sg->length,
+#endif
+ dir, caller);
+
+ return 0;
+}
+
+int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
struct scatterlist *sg;
@@ -203,7 +235,30 @@ int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_da
return 0;
}
-void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
+void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir, const char *caller)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct scatterlist *sg;
+ int i;
+
+ hdev->asic_funcs->dma_unmap_sgtable(hdev, sgt, dir);
+
+ if (trace_habanalabs_dma_unmap_page_enabled()) {
+ for_each_sgtable_dma_sg(sgt, sg, i)
+ trace_habanalabs_dma_unmap_page(hdev->dev, page_to_phys(sg_page(sg)),
+ sg->dma_address - prop->device_dma_offset_for_host_access,
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->dma_length,
+#else
+ sg->length,
+#endif
+ dir, caller);
+ }
+}
+
+void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
struct scatterlist *sg;
@@ -315,7 +370,9 @@ enum hl_device_status hl_device_status(struct hl_device *hdev)
{
enum hl_device_status status;
- if (hdev->reset_info.in_reset) {
+ if (hdev->device_fini_pending) {
+ status = HL_DEVICE_STATUS_MALFUNCTION;
+ } else if (hdev->reset_info.in_reset) {
if (hdev->reset_info.in_compute_reset)
status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE;
else
@@ -343,9 +400,9 @@ bool hl_device_operational(struct hl_device *hdev,
*status = current_status;
switch (current_status) {
+ case HL_DEVICE_STATUS_MALFUNCTION:
case HL_DEVICE_STATUS_IN_RESET:
case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
- case HL_DEVICE_STATUS_MALFUNCTION:
case HL_DEVICE_STATUS_NEEDS_RESET:
return false;
case HL_DEVICE_STATUS_OPERATIONAL:
@@ -406,8 +463,6 @@ static void hpriv_release(struct kref *ref)
hdev->asic_funcs->send_device_activity(hdev, false);
- put_pid(hpriv->taskpid);
-
hl_debugfs_remove_file(hpriv);
mutex_destroy(&hpriv->ctx_lock);
@@ -424,7 +479,7 @@ static void hpriv_release(struct kref *ref)
/* Check the device idle status and reset if not idle.
* Skip it if already in reset, or if device is going to be reset in any case.
*/
- if (!hdev->reset_info.in_reset && !reset_device && hdev->pdev && !hdev->pldm)
+ if (!hdev->reset_info.in_reset && !reset_device && !hdev->pldm)
device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask,
HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
if (!device_is_idle) {
@@ -446,14 +501,18 @@ static void hpriv_release(struct kref *ref)
list_del(&hpriv->dev_node);
mutex_unlock(&hdev->fpriv_list_lock);
+ put_pid(hpriv->taskpid);
+
if (reset_device) {
hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
} else {
/* Scrubbing is handled within hl_device_reset(), so here need to do it directly */
int rc = hdev->asic_funcs->scrub_device_mem(hdev);
- if (rc)
+ if (rc) {
dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
+ hl_device_reset(hdev, HL_DRV_RESET_HARD);
+ }
}
/* Now we can mark the compute_ctx as not active. Even if a reset is running in a different
@@ -516,24 +575,20 @@ static void print_device_in_use_info(struct hl_device *hdev, const char *message
}
/*
- * hl_device_release - release function for habanalabs device
- *
- * @inode: pointer to inode structure
- * @filp: pointer to file structure
+ * hl_device_release() - release function for habanalabs device.
+ * @ddev: pointer to DRM device structure.
+ * @file: pointer to DRM file private data structure.
*
* Called when process closes an habanalabs device
*/
-static int hl_device_release(struct inode *inode, struct file *filp)
+void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv)
{
- struct hl_fpriv *hpriv = filp->private_data;
- struct hl_device *hdev = hpriv->hdev;
-
- filp->private_data = NULL;
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
+ struct hl_device *hdev = to_hl_device(ddev);
if (!hdev) {
pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
put_pid(hpriv->taskpid);
- return 0;
}
hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
@@ -551,8 +606,6 @@ static int hl_device_release(struct inode *inode, struct file *filp)
}
hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
-
- return 0;
}
static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
@@ -571,11 +624,6 @@ static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
list_del(&hpriv->dev_node);
mutex_unlock(&hdev->fpriv_ctrl_list_lock);
out:
- /* release the eventfd */
- if (hpriv->notifier_event.eventfd)
- eventfd_ctx_put(hpriv->notifier_event.eventfd);
-
- mutex_destroy(&hpriv->notifier_event.lock);
put_pid(hpriv->taskpid);
kfree(hpriv);
@@ -583,18 +631,8 @@ out:
return 0;
}
-/*
- * hl_mmap - mmap function for habanalabs device
- *
- * @*filp: pointer to file structure
- * @*vma: pointer to vm_area_struct of the process
- *
- * Called when process does an mmap on habanalabs device. Call the relevant mmap
- * function at the end of the common code.
- */
-static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
+static int __hl_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
{
- struct hl_fpriv *hpriv = filp->private_data;
struct hl_device *hdev = hpriv->hdev;
unsigned long vm_pgoff;
@@ -617,14 +655,22 @@ static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
return -EINVAL;
}
-static const struct file_operations hl_ops = {
- .owner = THIS_MODULE,
- .open = hl_device_open,
- .release = hl_device_release,
- .mmap = hl_mmap,
- .unlocked_ioctl = hl_ioctl,
- .compat_ioctl = hl_ioctl
-};
+/*
+ * hl_mmap - mmap function for habanalabs device
+ *
+ * @*filp: pointer to file structure
+ * @*vma: pointer to vm_area_struct of the process
+ *
+ * Called when process does an mmap on habanalabs device. Call the relevant mmap
+ * function at the end of the common code.
+ */
+int hl_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ struct drm_file *file_priv = filp->private_data;
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
+
+ return __hl_mmap(hpriv, vma);
+}
static const struct file_operations hl_ctrl_ops = {
.owner = THIS_MODULE,
@@ -645,14 +691,14 @@ static void device_release_func(struct device *dev)
* @hdev: pointer to habanalabs device structure
* @class: pointer to the class object of the device
* @minor: minor number of the specific device
- * @fpos: file operations to install for this device
+ * @fops: file operations to install for this device
* @name: name of the device as it will appear in the filesystem
* @cdev: pointer to the char device object that will be initialized
* @dev: pointer to the device object that will be initialized
*
* Initialize a cdev and a Linux device for habanalabs's device.
*/
-static int device_init_cdev(struct hl_device *hdev, struct class *class,
+static int device_init_cdev(struct hl_device *hdev, const struct class *class,
int minor, const struct file_operations *fops,
char *name, struct cdev *cdev,
struct device **dev)
@@ -676,23 +722,26 @@ static int device_init_cdev(struct hl_device *hdev, struct class *class,
static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
{
+ const struct class *accel_class = hdev->drm.accel->kdev->class;
+ char name[32];
int rc;
- rc = cdev_device_add(&hdev->cdev, hdev->dev);
- if (rc) {
- dev_err(hdev->dev,
- "failed to add a char device to the system\n");
+ hdev->cdev_idx = hdev->drm.accel->index;
+
+ /* Initialize cdev and device structures for the control device */
+ snprintf(name, sizeof(name), "accel_controlD%d", hdev->cdev_idx);
+ rc = device_init_cdev(hdev, accel_class, hdev->cdev_idx, &hl_ctrl_ops, name,
+ &hdev->cdev_ctrl, &hdev->dev_ctrl);
+ if (rc)
return rc;
- }
rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
if (rc) {
- dev_err(hdev->dev,
- "failed to add a control char device to the system\n");
- goto delete_cdev_device;
+ dev_err(hdev->dev_ctrl,
+ "failed to add an accel control char device to the system\n");
+ goto free_ctrl_device;
}
- /* hl_sysfs_init() must be done after adding the device to the system */
rc = hl_sysfs_init(hdev);
if (rc) {
dev_err(hdev->dev, "failed to initialize sysfs\n");
@@ -707,23 +756,19 @@ static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
delete_ctrl_cdev_device:
cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
-delete_cdev_device:
- cdev_device_del(&hdev->cdev, hdev->dev);
+free_ctrl_device:
+ put_device(hdev->dev_ctrl);
return rc;
}
static void cdev_sysfs_debugfs_remove(struct hl_device *hdev)
{
if (!hdev->cdev_sysfs_debugfs_created)
- goto put_devices;
+ return;
- hl_debugfs_remove_device(hdev);
hl_sysfs_fini(hdev);
- cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
- cdev_device_del(&hdev->cdev, hdev->dev);
-put_devices:
- put_device(hdev->dev);
+ cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
put_device(hdev->dev_ctrl);
}
@@ -996,6 +1041,20 @@ static bool is_pci_link_healthy(struct hl_device *hdev)
return (vendor_id == PCI_VENDOR_ID_HABANALABS);
}
+static void hl_device_eq_heartbeat(struct hl_device *hdev)
+{
+ u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ if (!prop->cpucp_info.eq_health_check_supported)
+ return;
+
+ if (hdev->eq_heartbeat_received)
+ hdev->eq_heartbeat_received = false;
+ else
+ hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
+}
+
static void hl_device_heartbeat(struct work_struct *work)
{
struct hl_device *hdev = container_of(work, struct hl_device,
@@ -1003,9 +1062,16 @@ static void hl_device_heartbeat(struct work_struct *work)
struct hl_info_fw_err_info info = {0};
u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
- if (!hl_device_operational(hdev, NULL))
+ /* Start heartbeat checks only after driver has enabled events from FW */
+ if (!hl_device_operational(hdev, NULL) || !hdev->init_done)
goto reschedule;
+ /*
+ * For EQ health check need to check if driver received the heartbeat eq event
+ * in order to validate the eq is working.
+ */
+ hl_device_eq_heartbeat(hdev);
+
if (!hdev->asic_funcs->send_heartbeat(hdev))
goto reschedule;
@@ -1062,7 +1128,15 @@ static int device_late_init(struct hl_device *hdev)
hdev->high_pll = hdev->asic_prop.high_pll;
if (hdev->heartbeat) {
+ /*
+ * Before scheduling the heartbeat driver will check if eq event has received.
+ * for the first schedule we need to set the indication as true then for the next
+ * one this indication will be true only if eq event was sent by FW.
+ */
+ hdev->eq_heartbeat_received = true;
+
INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
+
schedule_delayed_work(&hdev->work_heartbeat,
usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
}
@@ -1302,18 +1376,18 @@ disable_device:
static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
{
struct task_struct *task = NULL;
- struct list_head *fd_list;
- struct hl_fpriv *hpriv;
- struct mutex *fd_lock;
+ struct list_head *hpriv_list;
+ struct hl_fpriv *hpriv;
+ struct mutex *hpriv_lock;
u32 pending_cnt;
- fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
- fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
+ hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
+ hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
/* Giving time for user to close FD, and for processes that are inside
* hl_device_open to finish
*/
- if (!list_empty(fd_list))
+ if (!list_empty(hpriv_list))
ssleep(1);
if (timeout) {
@@ -1329,12 +1403,12 @@ static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool
}
}
- mutex_lock(fd_lock);
+ mutex_lock(hpriv_lock);
/* This section must be protected because we are dereferencing
* pointers that are freed if the process exits
*/
- list_for_each_entry(hpriv, fd_list, dev_node) {
+ list_for_each_entry(hpriv, hpriv_list, dev_node) {
task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
if (task) {
dev_info(hdev->dev, "Killing user process pid=%d\n",
@@ -1344,17 +1418,13 @@ static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool
put_task_struct(task);
} else {
- /*
- * If we got here, it means that process was killed from outside the driver
- * right after it started looping on fd_list and before get_pid_task, thus
- * we don't need to kill it.
- */
dev_dbg(hdev->dev,
- "Can't get task struct for user process, assuming process was killed from outside the driver\n");
+ "Can't get task struct for user process %d, process was killed from outside the driver\n",
+ pid_nr(hpriv->taskpid));
}
}
- mutex_unlock(fd_lock);
+ mutex_unlock(hpriv_lock);
/*
* We killed the open users, but that doesn't mean they are closed.
@@ -1366,7 +1436,7 @@ static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool
*/
wait_for_processes:
- while ((!list_empty(fd_list)) && (pending_cnt)) {
+ while ((!list_empty(hpriv_list)) && (pending_cnt)) {
dev_dbg(hdev->dev,
"Waiting for all unmap operations to finish before hard reset\n");
@@ -1376,7 +1446,7 @@ wait_for_processes:
}
/* All processes exited successfully */
- if (list_empty(fd_list))
+ if (list_empty(hpriv_list))
return 0;
/* Give up waiting for processes to exit */
@@ -1390,17 +1460,17 @@ wait_for_processes:
static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
{
- struct list_head *fd_list;
+ struct list_head *hpriv_list;
struct hl_fpriv *hpriv;
- struct mutex *fd_lock;
+ struct mutex *hpriv_lock;
- fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
- fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
+ hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
+ hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
- mutex_lock(fd_lock);
- list_for_each_entry(hpriv, fd_list, dev_node)
+ mutex_lock(hpriv_lock);
+ list_for_each_entry(hpriv, hpriv_list, dev_node)
hpriv->hdev = NULL;
- mutex_unlock(fd_lock);
+ mutex_unlock(hpriv_lock);
}
static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
@@ -1916,7 +1986,16 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
}
ctx = hl_get_compute_ctx(hdev);
- if (!ctx || !ctx->hpriv->notifier_event.eventfd)
+ if (!ctx)
+ goto device_reset;
+
+ /*
+ * There is no point in postponing the reset if user is not registered for events.
+ * However if no eventfd_ctx exists but the device release watchdog is already scheduled, it
+ * just implies that user has unregistered as part of handling a previous event. In this
+ * case an immediate reset is not required.
+ */
+ if (!ctx->hpriv->notifier_event.eventfd && !hdev->reset_info.watchdog_active)
goto device_reset;
/* Schedule the device release watchdog work unless reset is already in progress or if the
@@ -1928,8 +2007,10 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
goto device_reset;
}
- if (hdev->reset_info.watchdog_active)
+ if (hdev->reset_info.watchdog_active) {
+ hdev->device_release_watchdog_work.flags |= flags;
goto out;
+ }
hdev->device_release_watchdog_work.flags = flags;
dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
@@ -1990,59 +2071,6 @@ void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
hl_notifier_event_send(&hpriv->notifier_event, event_mask);
mutex_unlock(&hdev->fpriv_list_lock);
-
- /* control device */
- mutex_lock(&hdev->fpriv_ctrl_list_lock);
-
- list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node)
- hl_notifier_event_send(&hpriv->notifier_event, event_mask);
-
- mutex_unlock(&hdev->fpriv_ctrl_list_lock);
-}
-
-static int create_cdev(struct hl_device *hdev)
-{
- char *name;
- int rc;
-
- hdev->cdev_idx = hdev->id / 2;
-
- name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx);
- if (!name) {
- rc = -ENOMEM;
- goto out_err;
- }
-
- /* Initialize cdev and device structures */
- rc = device_init_cdev(hdev, hdev->hclass, hdev->id, &hl_ops, name,
- &hdev->cdev, &hdev->dev);
-
- kfree(name);
-
- if (rc)
- goto out_err;
-
- name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx);
- if (!name) {
- rc = -ENOMEM;
- goto free_dev;
- }
-
- /* Initialize cdev and device structures for control device */
- rc = device_init_cdev(hdev, hdev->hclass, hdev->id_control, &hl_ctrl_ops,
- name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
-
- kfree(name);
-
- if (rc)
- goto free_dev;
-
- return 0;
-
-free_dev:
- put_device(hdev->dev);
-out_err:
- return rc;
}
/*
@@ -2057,16 +2085,14 @@ out_err:
int hl_device_init(struct hl_device *hdev)
{
int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
+ struct hl_ts_free_jobs *free_jobs_data;
bool expose_interfaces_on_err = false;
-
- rc = create_cdev(hdev);
- if (rc)
- goto out_disabled;
+ void *p;
/* Initialize ASIC function pointers and perform early init */
rc = device_early_init(hdev);
if (rc)
- goto free_dev;
+ goto out_disabled;
user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
hdev->asic_prop.user_interrupt_count;
@@ -2078,15 +2104,43 @@ int hl_device_init(struct hl_device *hdev)
rc = -ENOMEM;
goto early_fini;
}
+
+ /* Timestamp records supported only if CQ supported in device */
+ if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
+ for (i = 0 ; i < user_interrupt_cnt ; i++) {
+ p = vzalloc(TIMESTAMP_FREE_NODES_NUM *
+ sizeof(struct timestamp_reg_free_node));
+ if (!p) {
+ rc = -ENOMEM;
+ goto free_usr_intr_mem;
+ }
+ free_jobs_data = &hdev->user_interrupt[i].ts_free_jobs_data;
+ free_jobs_data->free_nodes_pool = p;
+ free_jobs_data->free_nodes_length = TIMESTAMP_FREE_NODES_NUM;
+ free_jobs_data->next_avail_free_node_idx = 0;
+ }
+ }
+ }
+
+ free_jobs_data = &hdev->common_user_cq_interrupt.ts_free_jobs_data;
+ p = vzalloc(TIMESTAMP_FREE_NODES_NUM *
+ sizeof(struct timestamp_reg_free_node));
+ if (!p) {
+ rc = -ENOMEM;
+ goto free_usr_intr_mem;
}
+ free_jobs_data->free_nodes_pool = p;
+ free_jobs_data->free_nodes_length = TIMESTAMP_FREE_NODES_NUM;
+ free_jobs_data->next_avail_free_node_idx = 0;
+
/*
* Start calling ASIC initialization. First S/W then H/W and finally
* late init
*/
rc = hdev->asic_funcs->sw_init(hdev);
if (rc)
- goto free_usr_intr_mem;
+ goto free_common_usr_intr_mem;
/* initialize completion structure for multi CS wait */
@@ -2253,6 +2307,14 @@ int hl_device_init(struct hl_device *hdev)
* From here there is no need to expose them in case of an error.
*/
expose_interfaces_on_err = false;
+
+ rc = drm_dev_register(&hdev->drm, 0);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to register DRM device, rc %d\n", rc);
+ rc = 0;
+ goto out_disabled;
+ }
+
rc = cdev_sysfs_debugfs_add(hdev);
if (rc) {
dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n");
@@ -2284,8 +2346,6 @@ int hl_device_init(struct hl_device *hdev)
"Successfully added device %s to habanalabs driver\n",
dev_name(&(hdev)->pdev->dev));
- hdev->init_done = true;
-
/* After initialization is done, we are ready to receive events from
* the F/W. We can't do it before because we will ignore events and if
* those events are fatal, we won't know about it and the device will
@@ -2293,6 +2353,8 @@ int hl_device_init(struct hl_device *hdev)
*/
hdev->asic_funcs->enable_events_from_fw(hdev);
+ hdev->init_done = true;
+
return 0;
cb_pool_fini:
@@ -2317,19 +2379,27 @@ hw_queues_destroy:
hl_hw_queues_destroy(hdev);
sw_fini:
hdev->asic_funcs->sw_fini(hdev);
+free_common_usr_intr_mem:
+ vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
free_usr_intr_mem:
- kfree(hdev->user_interrupt);
+ if (user_interrupt_cnt) {
+ for (i = 0 ; i < user_interrupt_cnt ; i++) {
+ if (!hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool)
+ break;
+ vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
+ }
+ kfree(hdev->user_interrupt);
+ }
early_fini:
device_early_fini(hdev);
-free_dev:
- put_device(hdev->dev_ctrl);
- put_device(hdev->dev);
out_disabled:
hdev->disabled = true;
- if (expose_interfaces_on_err)
+ if (expose_interfaces_on_err) {
+ drm_dev_register(&hdev->drm, 0);
cdev_sysfs_debugfs_add(hdev);
- dev_err(&hdev->pdev->dev,
- "Failed to initialize hl%d. Device %s is NOT usable !\n",
+ }
+
+ pr_err("Failed to initialize accel%d. Device %s is NOT usable!\n",
hdev->cdev_idx, dev_name(&hdev->pdev->dev));
return rc;
@@ -2344,12 +2414,13 @@ out_disabled:
*/
void hl_device_fini(struct hl_device *hdev)
{
+ u32 user_interrupt_cnt;
bool device_in_reset;
ktime_t timeout;
u64 reset_sec;
int i, rc;
- dev_info(hdev->dev, "Removing device\n");
+ dev_info(hdev->dev, "Removing device %s\n", dev_name(&(hdev)->pdev->dev));
hdev->device_fini_pending = 1;
flush_delayed_work(&hdev->device_reset_work.reset_work);
@@ -2425,14 +2496,14 @@ void hl_device_fini(struct hl_device *hdev)
hdev->process_kill_trial_cnt = 0;
rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false);
if (rc) {
- dev_crit(hdev->dev, "Failed to kill all open processes\n");
+ dev_crit(hdev->dev, "Failed to kill all open processes (%d)\n", rc);
device_disable_open_processes(hdev, false);
}
hdev->process_kill_trial_cnt = 0;
rc = device_kill_open_processes(hdev, 0, true);
if (rc) {
- dev_crit(hdev->dev, "Failed to kill all control device open processes\n");
+ dev_crit(hdev->dev, "Failed to kill all control device open processes (%d)\n", rc);
device_disable_open_processes(hdev, true);
}
@@ -2464,7 +2535,20 @@ void hl_device_fini(struct hl_device *hdev)
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
hl_cq_fini(hdev, &hdev->completion_queue[i]);
kfree(hdev->completion_queue);
- kfree(hdev->user_interrupt);
+
+ user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
+ hdev->asic_prop.user_interrupt_count;
+
+ if (user_interrupt_cnt) {
+ if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
+ for (i = 0 ; i < user_interrupt_cnt ; i++)
+ vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
+ }
+
+ kfree(hdev->user_interrupt);
+ }
+
+ vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
hl_hw_queues_destroy(hdev);
@@ -2475,6 +2559,7 @@ void hl_device_fini(struct hl_device *hdev)
/* Hide devices and sysfs/debugfs files from user */
cdev_sysfs_debugfs_remove(hdev);
+ drm_dev_unregister(&hdev->drm);
hl_debugfs_device_fini(hdev);
@@ -2690,6 +2775,20 @@ void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
*info->event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
}
+void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count)
+{
+ struct engine_err_info *info = &hdev->captured_err_info.engine_err;
+
+ /* Capture only the first engine error */
+ if (atomic_cmpxchg(&info->event_detected, 0, 1))
+ return;
+
+ info->event.timestamp = ktime_to_ns(ktime_get());
+ info->event.engine_id = engine_id;
+ info->event.error_count = error_count;
+ info->event_info_available = true;
+}
+
void hl_enable_err_info_capture(struct hl_error_info *captured_err_info)
{
vfree(captured_err_info->page_fault_info.user_mappings);
diff --git a/drivers/accel/habanalabs/common/firmware_if.c b/drivers/accel/habanalabs/common/firmware_if.c
index acbc1a6b5cb1..47e8384134aa 100644
--- a/drivers/accel/habanalabs/common/firmware_if.c
+++ b/drivers/accel/habanalabs/common/firmware_if.c
@@ -6,7 +6,7 @@
*/
#include "habanalabs.h"
-#include "../include/common/hl_boot_if.h"
+#include <linux/habanalabs/hl_boot_if.h>
#include <linux/firmware.h>
#include <linux/crc32.h>
@@ -724,6 +724,11 @@ static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
err_exists = true;
}
+ if (err_val & CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL) {
+ dev_err(hdev->dev, "Device boot error - Failed to set threshold for temperature sensor\n");
+ err_exists = true;
+ }
+
if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
/* Ignore this bit, don't prevent driver loading */
dev_dbg(hdev->dev, "device unusable status is set\n");
@@ -1459,6 +1464,10 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
dev_err(hdev->dev,
"Device boot progress - Stuck in preboot after security initialization\n");
break;
+ case CPU_BOOT_STATUS_FW_SHUTDOWN_PREP:
+ dev_err(hdev->dev,
+ "Device boot progress - Stuck in preparation for shutdown\n");
+ break;
default:
dev_err(hdev->dev,
"Device boot progress - Invalid or unexpected status code %d\n", status);
@@ -1469,8 +1478,9 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
int hl_fw_wait_preboot_ready(struct hl_device *hdev)
{
struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;
- u32 status;
- int rc;
+ u32 status = 0, timeout;
+ int rc, tries = 1;
+ bool preboot_still_runs;
/* Need to check two possible scenarios:
*
@@ -1480,6 +1490,8 @@ int hl_fw_wait_preboot_ready(struct hl_device *hdev)
* All other status values - for older firmwares where the uboot was
* loaded from the FLASH
*/
+ timeout = pre_fw_load->wait_for_preboot_timeout;
+retry:
rc = hl_poll_timeout(
hdev,
pre_fw_load->cpu_boot_status_reg,
@@ -1488,7 +1500,24 @@ int hl_fw_wait_preboot_ready(struct hl_device *hdev)
(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
(status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
hdev->fw_poll_interval_usec,
- pre_fw_load->wait_for_preboot_timeout);
+ timeout);
+ /*
+ * if F/W reports "security-ready" it means preboot might take longer.
+ * If the field 'wait_for_preboot_extended_timeout' is non 0 we wait again
+ * with that timeout
+ */
+ preboot_still_runs = (status == CPU_BOOT_STATUS_SECURITY_READY ||
+ status == CPU_BOOT_STATUS_IN_PREBOOT ||
+ status == CPU_BOOT_STATUS_FW_SHUTDOWN_PREP ||
+ status == CPU_BOOT_STATUS_DRAM_RDY);
+
+ if (rc && tries && preboot_still_runs) {
+ tries--;
+ if (pre_fw_load->wait_for_preboot_extended_timeout) {
+ timeout = pre_fw_load->wait_for_preboot_extended_timeout;
+ goto retry;
+ }
+ }
if (rc) {
detect_cpu_boot_status(hdev, status);
@@ -2743,7 +2772,8 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
struct lkd_fw_binning_info *binning_info;
- rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
+ rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
+ sizeof(struct lkd_msg_comms));
if (rc)
goto protocol_err;
@@ -2777,6 +2807,11 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
hdev->decoder_binning, hdev->rotator_binning);
}
+ if (hdev->asic_prop.support_dynamic_resereved_fw_size) {
+ hdev->asic_prop.reserved_fw_mem_size =
+ le32_to_cpu(fw_loader->dynamic_loader.comm_desc.rsvd_mem_size_mb);
+ }
+
return 0;
}
diff --git a/drivers/accel/habanalabs/common/habanalabs.h b/drivers/accel/habanalabs/common/habanalabs.h
index 2f027d5a8206..1655c101c705 100644
--- a/drivers/accel/habanalabs/common/habanalabs.h
+++ b/drivers/accel/habanalabs/common/habanalabs.h
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0
*
- * Copyright 2016-2022 HabanaLabs, Ltd.
+ * Copyright 2016-2023 HabanaLabs, Ltd.
* All Rights Reserved.
*
*/
@@ -8,7 +8,7 @@
#ifndef HABANALABSP_H_
#define HABANALABSP_H_
-#include "../include/common/cpucp_if.h"
+#include <linux/habanalabs/cpucp_if.h>
#include "../include/common/qman_if.h"
#include "../include/hw_ip/mmu/mmu_general.h"
#include <uapi/drm/habanalabs_accel.h>
@@ -29,6 +29,9 @@
#include <linux/coresight.h>
#include <linux/dma-buf.h>
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+
#include "security.h"
#define HL_NAME "habanalabs"
@@ -82,8 +85,6 @@ struct hl_fpriv;
#define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */
-#define HL_SIM_MAX_TIMEOUT_US 100000000 /* 100s */
-
#define HL_INVALID_QUEUE UINT_MAX
#define HL_COMMON_USER_CQ_INTERRUPT_ID 0xFFF
@@ -103,6 +104,8 @@ struct hl_fpriv;
/* MMU */
#define MMU_HASH_TABLE_BITS 7 /* 1 << 7 buckets */
+#define TIMESTAMP_FREE_NODES_NUM 512
+
/**
* enum hl_mmu_page_table_location - mmu page table location
* @MMU_DR_PGT: page-table is located on device DRAM.
@@ -154,6 +157,11 @@ enum hl_mmu_page_table_location {
#define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \
hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__)
+#define hl_dma_map_sgtable(hdev, sgt, dir) \
+ hl_dma_map_sgtable_caller(hdev, sgt, dir, __func__)
+#define hl_dma_unmap_sgtable(hdev, sgt, dir) \
+ hl_dma_unmap_sgtable_caller(hdev, sgt, dir, __func__)
+
/*
* Reset Flags
*
@@ -545,8 +553,7 @@ struct hl_hints_range {
* allocated with huge pages.
* @hints_dram_reserved_va_range: dram hint addresses reserved range.
* @hints_host_reserved_va_range: host hint addresses reserved range.
- * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved
- * range.
+ * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved range.
* @sram_base_address: SRAM physical start address.
* @sram_end_address: SRAM physical end address.
* @sram_user_base_address - SRAM physical start address for user access.
@@ -585,7 +592,7 @@ struct hl_hints_range {
* @mmu_pte_size: PTE size in MMU page tables.
* @mmu_hop_table_size: MMU hop table size.
* @mmu_hop0_tables_total_size: total size of MMU hop0 tables.
- * @dram_page_size: page size for MMU DRAM allocation.
+ * @dram_page_size: The DRAM physical page size.
* @cfg_size: configuration space size on SRAM.
* @sram_size: total size of SRAM.
* @max_asid: maximum number of open contexts (ASIDs).
@@ -641,6 +648,7 @@ struct hl_hints_range {
* @glbl_err_cause_num: global err cause number.
* @hbw_flush_reg: register to read to generate HBW flush. value of 0 means HBW flush is
* not supported.
+ * @reserved_fw_mem_size: size in MB of dram memory reserved for FW.
* @collective_first_sob: first sync object available for collective use
* @collective_first_mon: first monitor available for collective use
* @sync_stream_first_sob: first sync object available for sync stream use
@@ -686,9 +694,10 @@ struct hl_hints_range {
* @configurable_stop_on_err: is stop-on-error option configurable via debugfs.
* @set_max_power_on_device_init: true if need to set max power in F/W on device init.
* @supports_user_set_page_size: true if user can set the allocation page size.
- * @dma_mask: the dma mask to be set for this device
+ * @dma_mask: the dma mask to be set for this device.
* @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported.
* @supports_engine_modes: true if changing engines/engine_cores modes is supported.
+ * @support_dynamic_resereved_fw_size: true if we support dynamic reserved size for fw.
*/
struct asic_fixed_properties {
struct hw_queue_properties *hw_queues_props;
@@ -772,6 +781,7 @@ struct asic_fixed_properties {
u32 num_of_special_blocks;
u32 glbl_err_cause_num;
u32 hbw_flush_reg;
+ u32 reserved_fw_mem_size;
u16 collective_first_sob;
u16 collective_first_mon;
u16 sync_stream_first_sob;
@@ -808,6 +818,7 @@ struct asic_fixed_properties {
u8 dma_mask;
u8 supports_advanced_cpucp_rc;
u8 supports_engine_modes;
+ u8 support_dynamic_resereved_fw_size;
};
/**
@@ -1098,19 +1109,41 @@ enum hl_user_interrupt_type {
};
/**
+ * struct hl_ts_free_jobs - holds user interrupt ts free nodes related data
+ * @free_nodes_pool: pool of nodes to be used for free timestamp jobs
+ * @free_nodes_length: number of nodes in free_nodes_pool
+ * @next_avail_free_node_idx: index of the next free node in the pool
+ *
+ * the free nodes pool must be protected by the user interrupt lock
+ * to avoid race between different interrupts which are using the same
+ * ts buffer with different offsets.
+ */
+struct hl_ts_free_jobs {
+ struct timestamp_reg_free_node *free_nodes_pool;
+ u32 free_nodes_length;
+ u32 next_avail_free_node_idx;
+};
+
+/**
* struct hl_user_interrupt - holds user interrupt information
* @hdev: pointer to the device structure
+ * @ts_free_jobs_data: timestamp free jobs related data
* @type: user interrupt type
* @wait_list_head: head to the list of user threads pending on this interrupt
+ * @ts_list_head: head to the list of timestamp records
* @wait_list_lock: protects wait_list_head
+ * @ts_list_lock: protects ts_list_head
* @timestamp: last timestamp taken upon interrupt
* @interrupt_id: msix interrupt id
*/
struct hl_user_interrupt {
struct hl_device *hdev;
+ struct hl_ts_free_jobs ts_free_jobs_data;
enum hl_user_interrupt_type type;
struct list_head wait_list_head;
+ struct list_head ts_list_head;
spinlock_t wait_list_lock;
+ spinlock_t ts_list_lock;
ktime_t timestamp;
u32 interrupt_id;
};
@@ -1120,11 +1153,15 @@ struct hl_user_interrupt {
* @free_objects_node: node in the list free_obj_jobs
* @cq_cb: pointer to cq command buffer to be freed
* @buf: pointer to timestamp buffer to be freed
+ * @in_use: indicates whether the node still in use in workqueue thread.
+ * @dynamic_alloc: indicates whether the node was allocated dynamically in the interrupt handler
*/
struct timestamp_reg_free_node {
struct list_head free_objects_node;
struct hl_cb *cq_cb;
struct hl_mmap_mem_buf *buf;
+ atomic_t in_use;
+ u8 dynamic_alloc;
};
/* struct timestamp_reg_work_obj - holds the timestamp registration free objects job
@@ -1133,17 +1170,21 @@ struct timestamp_reg_free_node {
* @free_obj: workqueue object to free timestamp registration node objects
* @hdev: pointer to the device structure
* @free_obj_head: list of free jobs nodes (node type timestamp_reg_free_node)
+ * @dynamic_alloc_free_obj_head: list of free jobs nodes which were dynamically allocated in the
+ * interrupt handler.
*/
struct timestamp_reg_work_obj {
struct work_struct free_obj;
struct hl_device *hdev;
struct list_head *free_obj_head;
+ struct list_head *dynamic_alloc_free_obj_head;
};
/* struct timestamp_reg_info - holds the timestamp registration related data.
* @buf: pointer to the timestamp buffer which include both user/kernel buffers.
* relevant only when doing timestamps records registration.
* @cq_cb: pointer to CQ counter CB.
+ * @interrupt: interrupt that the node hanged on it's wait list.
* @timestamp_kernel_addr: timestamp handle address, where to set timestamp
* relevant only when doing timestamps records
* registration.
@@ -1153,17 +1194,18 @@ struct timestamp_reg_work_obj {
* allocating records dynamically.
*/
struct timestamp_reg_info {
- struct hl_mmap_mem_buf *buf;
- struct hl_cb *cq_cb;
- u64 *timestamp_kernel_addr;
- u8 in_use;
+ struct hl_mmap_mem_buf *buf;
+ struct hl_cb *cq_cb;
+ struct hl_user_interrupt *interrupt;
+ u64 *timestamp_kernel_addr;
+ bool in_use;
};
/**
* struct hl_user_pending_interrupt - holds a context to a user thread
* pending on an interrupt
* @ts_reg_info: holds the timestamps registration nodes info
- * @wait_list_node: node in the list of user threads pending on an interrupt
+ * @list_node: node in the list of user threads pending on an interrupt or timestamp
* @fence: hl fence object for interrupt completion
* @cq_target_value: CQ target value
* @cq_kernel_addr: CQ kernel address, to be used in the cq interrupt
@@ -1171,7 +1213,7 @@ struct timestamp_reg_info {
*/
struct hl_user_pending_interrupt {
struct timestamp_reg_info ts_reg_info;
- struct list_head wait_list_node;
+ struct list_head list_node;
struct hl_fence fence;
u64 cq_target_value;
u64 *cq_kernel_addr;
@@ -1370,6 +1412,8 @@ struct dynamic_fw_load_mgr {
* @boot_err0_reg: boot_err0 register address
* @boot_err1_reg: boot_err1 register address
* @wait_for_preboot_timeout: timeout to poll for preboot ready
+ * @wait_for_preboot_extended_timeout: timeout to pull for preboot ready in case where we know
+ * preboot needs longer time.
*/
struct pre_fw_load_props {
u32 cpu_boot_status_reg;
@@ -1378,6 +1422,7 @@ struct pre_fw_load_props {
u32 boot_err0_reg;
u32 boot_err1_reg;
u32 wait_for_preboot_timeout;
+ u32 wait_for_preboot_extended_timeout;
};
/**
@@ -1477,11 +1522,9 @@ struct engines_data {
* @asic_dma_pool_free: free small DMA allocation from pool.
* @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
* @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
- * @asic_dma_unmap_single: unmap a single DMA buffer
- * @asic_dma_map_single: map a single buffer to a DMA
- * @hl_dma_unmap_sgtable: DMA unmap scatter-gather table.
+ * @dma_unmap_sgtable: DMA unmap scatter-gather table.
+ * @dma_map_sgtable: DMA map scatter-gather table.
* @cs_parser: parse Command Submission.
- * @asic_dma_map_sgtable: DMA map scatter-gather table.
* @add_end_of_cb_packets: Add packets to the end of CB, if device requires it.
* @update_eq_ci: update event queue CI.
* @context_switch: called upon ASID context switch.
@@ -1602,18 +1645,11 @@ struct hl_asic_funcs {
size_t size, dma_addr_t *dma_handle);
void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
size_t size, void *vaddr);
- void (*asic_dma_unmap_single)(struct hl_device *hdev,
- dma_addr_t dma_addr, int len,
- enum dma_data_direction dir);
- dma_addr_t (*asic_dma_map_single)(struct hl_device *hdev,
- void *addr, int len,
+ void (*dma_unmap_sgtable)(struct hl_device *hdev, struct sg_table *sgt,
enum dma_data_direction dir);
- void (*hl_dma_unmap_sgtable)(struct hl_device *hdev,
- struct sg_table *sgt,
+ int (*dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt,
enum dma_data_direction dir);
int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
- int (*asic_dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt,
- enum dma_data_direction dir);
void (*add_end_of_cb_packets)(struct hl_device *hdev,
void *kernel_address, u32 len,
u32 original_len,
@@ -1771,16 +1807,19 @@ struct hl_cs_counters_atomic {
* @phys_pg_pack: pointer to physical page pack if the dma-buf was exported
* where virtual memory is supported.
* @memhash_hnode: pointer to the memhash node. this object holds the export count.
- * @device_address: physical address of the device's memory. Relevant only
- * if phys_pg_pack is NULL (dma-buf was exported from address).
- * The total size can be taken from the dmabuf object.
+ * @offset: the offset into the buffer from which the memory is exported.
+ * Relevant only if virtual memory is supported and phys_pg_pack is being used.
+ * device_phys_addr: physical address of the device's memory. Relevant only
+ * if phys_pg_pack is NULL (dma-buf was exported from address).
+ * The total size can be taken from the dmabuf object.
*/
struct hl_dmabuf_priv {
struct dma_buf *dmabuf;
struct hl_ctx *ctx;
struct hl_vm_phys_pg_pack *phys_pg_pack;
struct hl_vm_hash_node *memhash_hnode;
- uint64_t device_address;
+ u64 offset;
+ u64 device_phys_addr;
};
#define HL_CS_OUTCOME_HISTORY_LEN 256
@@ -1835,6 +1874,7 @@ struct hl_cs_outcome_store {
* @va_range: holds available virtual addresses for host and dram mappings.
* @mem_hash_lock: protects the mem_hash.
* @hw_block_list_lock: protects the HW block memory list.
+ * @ts_reg_lock: timestamp registration ioctls lock.
* @debugfs_list: node in debugfs list of contexts.
* @hw_block_mem_list: list of HW block virtual mapped addresses.
* @cs_counters: context command submission counters.
@@ -1871,6 +1911,7 @@ struct hl_ctx {
struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX];
struct mutex mem_hash_lock;
struct mutex hw_block_list_lock;
+ struct mutex ts_reg_lock;
struct list_head debugfs_list;
struct list_head hw_block_mem_list;
struct hl_cs_counters_atomic cs_counters;
@@ -1917,17 +1958,17 @@ struct hl_ctx_mgr {
* @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
*/
struct hl_userptr {
- enum vm_type vm_type; /* must be first */
- struct list_head job_node;
- struct page **pages;
- unsigned int npages;
- struct sg_table *sgt;
- enum dma_data_direction dir;
- struct list_head debugfs_list;
- pid_t pid;
- u64 addr;
- u64 size;
- u8 dma_mapped;
+ enum vm_type vm_type; /* must be first */
+ struct list_head job_node;
+ struct page **pages;
+ unsigned int npages;
+ struct sg_table *sgt;
+ enum dma_data_direction dir;
+ struct list_head debugfs_list;
+ pid_t pid;
+ u64 addr;
+ u64 size;
+ u8 dma_mapped;
};
/**
@@ -2148,7 +2189,6 @@ struct hl_vm_hw_block_list_node {
* @pages: the physical page array.
* @npages: num physical pages in the pack.
* @total_size: total size of all the pages in this list.
- * @exported_size: buffer exported size.
* @node: used to attach to deletion list that is used when all the allocations are cleared
* at the teardown of the context.
* @mapping_cnt: number of shared mappings.
@@ -2165,7 +2205,6 @@ struct hl_vm_phys_pg_pack {
u64 *pages;
u64 npages;
u64 total_size;
- u64 exported_size;
struct list_head node;
atomic_t mapping_cnt;
u32 asid;
@@ -2250,7 +2289,7 @@ struct hl_notifier_event {
/**
* struct hl_fpriv - process information stored in FD private data.
* @hdev: habanalabs device structure.
- * @filp: pointer to the given file structure.
+ * @file_priv: pointer to the DRM file private data structure.
* @taskpid: current process ID.
* @ctx: current executing context. TODO: remove for multiple ctx per process
* @ctx_mgr: context manager to handle multiple context for this FD.
@@ -2265,7 +2304,7 @@ struct hl_notifier_event {
*/
struct hl_fpriv {
struct hl_device *hdev;
- struct file *filp;
+ struct drm_file *file_priv;
struct pid *taskpid;
struct hl_ctx *ctx;
struct hl_ctx_mgr ctx_mgr;
@@ -2706,6 +2745,8 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
usr_intr.type = intr_type; \
INIT_LIST_HEAD(&usr_intr.wait_list_head); \
spin_lock_init(&usr_intr.wait_list_lock); \
+ INIT_LIST_HEAD(&usr_intr.ts_list_head); \
+ spin_lock_init(&usr_intr.ts_list_lock); \
})
struct hwmon_chip_info;
@@ -3055,6 +3096,20 @@ struct fw_err_info {
};
/**
+ * struct engine_err_info - engine error information.
+ * @event: holds information on the event.
+ * @event_detected: if set as 1, then an engine event was discovered for the
+ * first time after the driver has finished booting-up.
+ * @event_info_available: indicates that an engine event info is now available.
+ */
+struct engine_err_info {
+ struct hl_info_engine_err_event event;
+ atomic_t event_detected;
+ bool event_info_available;
+};
+
+
+/**
* struct hl_error_info - holds information collected during an error.
* @cs_timeout: CS timeout error information.
* @razwi_info: RAZWI information.
@@ -3062,6 +3117,7 @@ struct fw_err_info {
* @page_fault_info: page fault information.
* @hw_err: (fatal) hardware error information.
* @fw_err: firmware error information.
+ * @engine_err: engine error information.
*/
struct hl_error_info {
struct cs_timeout_info cs_timeout;
@@ -3070,6 +3126,7 @@ struct hl_error_info {
struct page_fault_info page_fault_info;
struct hw_err_info hw_err;
struct fw_err_info fw_err;
+ struct engine_err_info engine_err;
};
/**
@@ -3117,8 +3174,7 @@ struct hl_reset_info {
* (required only for PCI address match mode)
* @pcie_bar: array of available PCIe bars virtual addresses.
* @rmmio: configuration area address on SRAM.
- * @hclass: pointer to the habanalabs class.
- * @cdev: related char device.
+ * @drm: related DRM device.
* @cdev_ctrl: char device for control operations only (INFO IOCTL)
* @dev: related kernel basic device structure.
* @dev_ctrl: related kernel device structure for the control device
@@ -3245,8 +3301,7 @@ struct hl_reset_info {
* @rotator_binning: contains mask of rotators engines that is received from the f/w
* which indicates which rotator engines are binned-out(Gaudi3 and above).
* @id: device minor.
- * @id_control: minor of the control device.
- * @cdev_idx: char device index. Used for setting its name.
+ * @cdev_idx: char device index.
* @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
* addresses.
* @is_in_dram_scrub: true if dram scrub operation is on going.
@@ -3289,6 +3344,7 @@ struct hl_reset_info {
* device.
* @supports_ctx_switch: true if a ctx switch is required upon first submission.
* @support_preboot_binning: true if we support read binning info from preboot.
+ * @eq_heartbeat_received: indication that eq heartbeat event has received from FW.
* @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing.
* @fw_components: Controls which f/w components to load to the device. There are multiple f/w
* stages and sometimes we want to stop at a certain stage. Used only for testing.
@@ -3308,8 +3364,7 @@ struct hl_device {
u64 pcie_bar_phys[HL_PCI_NUM_BARS];
void __iomem *pcie_bar[HL_PCI_NUM_BARS];
void __iomem *rmmio;
- struct class *hclass;
- struct cdev cdev;
+ struct drm_device drm;
struct cdev cdev_ctrl;
struct device *dev;
struct device *dev_ctrl;
@@ -3418,7 +3473,6 @@ struct hl_device {
u32 device_release_watchdog_timeout_sec;
u32 rotator_binning;
u16 id;
- u16 id_control;
u16 cdev_idx;
u16 cpu_pci_msb_addr;
u8 is_in_dram_scrub;
@@ -3451,6 +3505,7 @@ struct hl_device {
u8 reset_upon_device_release;
u8 supports_ctx_switch;
u8 support_preboot_binning;
+ u8 eq_heartbeat_received;
/* Parameters for bring-up to be upstreamed */
u64 nic_ports_mask;
@@ -3582,6 +3637,11 @@ static inline bool hl_mem_area_inside_range(u64 address, u64 size,
return false;
}
+static inline struct hl_device *to_hl_device(struct drm_device *ddev)
+{
+ return container_of(ddev, struct hl_device, drm);
+}
+
/**
* hl_mem_area_crosses_range() - Checks whether address+size crossing a range.
* @address: The start address of the area we want to validate.
@@ -3611,8 +3671,13 @@ void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t
dma_addr_t *dma_handle, const char *caller);
void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
const char *caller);
-int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir);
-void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
+int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir, const char *caller);
+void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir, const char *caller);
+int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt,
+ enum dma_data_direction dir);
+void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
enum dma_data_direction dir);
int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar);
@@ -3620,7 +3685,12 @@ int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
enum debugfs_access_type acc_type);
int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
u64 addr, u64 *val, enum debugfs_access_type acc_type);
-int hl_device_open(struct inode *inode, struct file *filp);
+
+int hl_mmap(struct file *filp, struct vm_area_struct *vma);
+
+int hl_device_open(struct drm_device *drm, struct drm_file *file_priv);
+void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv);
+
int hl_device_open_ctrl(struct inode *inode, struct file *filp);
bool hl_device_operational(struct hl_device *hdev,
enum hl_device_status *status);
@@ -3652,8 +3722,9 @@ void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
irqreturn_t hl_irq_handler_cq(int irq, void *arg);
irqreturn_t hl_irq_handler_eq(int irq, void *arg);
irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg);
-irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg);
+irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg);
irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg);
+irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg);
u32 hl_cq_inc_ptr(u32 ptr);
int hl_asid_init(struct hl_device *hdev);
@@ -3944,16 +4015,14 @@ void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_
u64 *event_mask);
void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask);
void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info);
+void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count);
void hl_enable_err_info_capture(struct hl_error_info *captured_err_info);
#ifdef CONFIG_DEBUG_FS
-void hl_debugfs_init(void);
-void hl_debugfs_fini(void);
int hl_debugfs_device_init(struct hl_device *hdev);
void hl_debugfs_device_fini(struct hl_device *hdev);
void hl_debugfs_add_device(struct hl_device *hdev);
-void hl_debugfs_remove_device(struct hl_device *hdev);
void hl_debugfs_add_file(struct hl_fpriv *hpriv);
void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
void hl_debugfs_add_cb(struct hl_cb *cb);
@@ -3972,14 +4041,6 @@ void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
#else
-static inline void __init hl_debugfs_init(void)
-{
-}
-
-static inline void hl_debugfs_fini(void)
-{
-}
-
static inline int hl_debugfs_device_init(struct hl_device *hdev)
{
return 0;
@@ -3993,10 +4054,6 @@ static inline void hl_debugfs_add_device(struct hl_device *hdev)
{
}
-static inline void hl_debugfs_remove_device(struct hl_device *hdev)
-{
-}
-
static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
{
}
@@ -4108,11 +4165,12 @@ void hl_ack_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset,
const u32 pb_blocks[], u32 blocks_array_size);
/* IOCTLs */
-long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
-int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data);
-int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data);
-int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data);
-int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data);
+int hl_info_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
+int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
+int hl_cs_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
+int hl_wait_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
+int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
+int hl_debug_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
#endif /* HABANALABSP_H_ */
diff --git a/drivers/accel/habanalabs/common/habanalabs_drv.c b/drivers/accel/habanalabs/common/habanalabs_drv.c
index 7263e84c1a4d..306a5bc9bf89 100644
--- a/drivers/accel/habanalabs/common/habanalabs_drv.c
+++ b/drivers/accel/habanalabs/common/habanalabs_drv.c
@@ -14,6 +14,11 @@
#include <linux/pci.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
+#include <linux/version.h>
+
+#include <drm/drm_accel.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
#define CREATE_TRACE_POINTS
#include <trace/events/habanalabs.h>
@@ -27,7 +32,6 @@ MODULE_DESCRIPTION(HL_DRIVER_DESC);
MODULE_LICENSE("GPL v2");
static int hl_major;
-static struct class *hl_class;
static DEFINE_IDR(hl_devs_idr);
static DEFINE_MUTEX(hl_devs_idr_lock);
@@ -70,6 +74,42 @@ static const struct pci_device_id ids[] = {
};
MODULE_DEVICE_TABLE(pci, ids);
+static const struct drm_ioctl_desc hl_drm_ioctls[] = {
+ DRM_IOCTL_DEF_DRV(HL_INFO, hl_info_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(HL_CB, hl_cb_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(HL_CS, hl_cs_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(HL_WAIT_CS, hl_wait_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(HL_MEMORY, hl_mem_ioctl, 0),
+ DRM_IOCTL_DEF_DRV(HL_DEBUG, hl_debug_ioctl, 0),
+};
+
+static const struct file_operations hl_fops = {
+ .owner = THIS_MODULE,
+ .open = accel_open,
+ .release = drm_release,
+ .unlocked_ioctl = drm_ioctl,
+ .compat_ioctl = drm_compat_ioctl,
+ .llseek = noop_llseek,
+ .mmap = hl_mmap
+};
+
+static const struct drm_driver hl_driver = {
+ .driver_features = DRIVER_COMPUTE_ACCEL,
+
+ .name = HL_NAME,
+ .desc = HL_DRIVER_DESC,
+ .major = LINUX_VERSION_MAJOR,
+ .minor = LINUX_VERSION_PATCHLEVEL,
+ .patchlevel = LINUX_VERSION_SUBLEVEL,
+ .date = "20190505",
+
+ .fops = &hl_fops,
+ .open = hl_device_open,
+ .postclose = hl_device_release,
+ .ioctls = hl_drm_ioctls,
+ .num_ioctls = ARRAY_SIZE(hl_drm_ioctls)
+};
+
/*
* get_asic_type - translate device id to asic type
*
@@ -123,43 +163,28 @@ static bool is_asic_secured(enum hl_asic_type asic_type)
}
/*
- * hl_device_open - open function for habanalabs device
- *
- * @inode: pointer to inode structure
- * @filp: pointer to file structure
+ * hl_device_open() - open function for habanalabs device.
+ * @ddev: pointer to DRM device structure.
+ * @file: pointer to DRM file private data structure.
*
* Called when process opens an habanalabs device.
*/
-int hl_device_open(struct inode *inode, struct file *filp)
+int hl_device_open(struct drm_device *ddev, struct drm_file *file_priv)
{
+ struct hl_device *hdev = to_hl_device(ddev);
enum hl_device_status status;
- struct hl_device *hdev;
struct hl_fpriv *hpriv;
int rc;
- mutex_lock(&hl_devs_idr_lock);
- hdev = idr_find(&hl_devs_idr, iminor(inode));
- mutex_unlock(&hl_devs_idr_lock);
-
- if (!hdev) {
- pr_err("Couldn't find device %d:%d\n",
- imajor(inode), iminor(inode));
- return -ENXIO;
- }
-
hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
if (!hpriv)
return -ENOMEM;
hpriv->hdev = hdev;
- filp->private_data = hpriv;
- hpriv->filp = filp;
-
mutex_init(&hpriv->notifier_event.lock);
mutex_init(&hpriv->restore_phase_mutex);
mutex_init(&hpriv->ctx_lock);
kref_init(&hpriv->refcount);
- nonseekable_open(inode, filp);
hl_ctx_mgr_init(&hpriv->ctx_mgr);
hl_mem_mgr_init(hpriv->hdev->dev, &hpriv->mem_mgr);
@@ -225,6 +250,9 @@ int hl_device_open(struct inode *inode, struct file *filp)
hdev->last_successful_open_jif = jiffies;
hdev->last_successful_open_ktime = ktime_get();
+ file_priv->driver_priv = hpriv;
+ hpriv->file_priv = file_priv;
+
return 0;
out_err:
@@ -232,7 +260,6 @@ out_err:
hl_mem_mgr_fini(&hpriv->mem_mgr);
hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
- filp->private_data = NULL;
mutex_destroy(&hpriv->ctx_lock);
mutex_destroy(&hpriv->restore_phase_mutex);
mutex_destroy(&hpriv->notifier_event.lock);
@@ -268,9 +295,7 @@ int hl_device_open_ctrl(struct inode *inode, struct file *filp)
*/
hpriv->hdev = hdev;
filp->private_data = hpriv;
- hpriv->filp = filp;
- mutex_init(&hpriv->notifier_event.lock);
nonseekable_open(inode, filp);
hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
@@ -317,7 +342,6 @@ static void copy_kernel_module_params_to_device(struct hl_device *hdev)
hdev->asic_prop.fw_security_enabled = is_asic_secured(hdev->asic_type);
hdev->major = hl_major;
- hdev->hclass = hl_class;
hdev->memory_scrub = memory_scrub;
hdev->reset_on_lockup = reset_on_lockup;
hdev->boot_error_status_mask = boot_error_status_mask;
@@ -383,6 +407,31 @@ static int fixup_device_params(struct hl_device *hdev)
return 0;
}
+static int allocate_device_id(struct hl_device *hdev)
+{
+ int id;
+
+ mutex_lock(&hl_devs_idr_lock);
+ id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS, GFP_KERNEL);
+ mutex_unlock(&hl_devs_idr_lock);
+
+ if (id < 0) {
+ if (id == -ENOSPC)
+ pr_err("too many devices in the system\n");
+ return -EBUSY;
+ }
+
+ hdev->id = id;
+
+ /*
+ * Firstly initialized with the internal device ID.
+ * Will be updated later after the DRM device registration to hold the minor ID.
+ */
+ hdev->cdev_idx = hdev->id;
+
+ return 0;
+}
+
/**
* create_hdev - create habanalabs device instance
*
@@ -395,27 +444,29 @@ static int fixup_device_params(struct hl_device *hdev)
*/
static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
{
- int main_id, ctrl_id = 0, rc = 0;
struct hl_device *hdev;
+ int rc;
*dev = NULL;
- hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
- if (!hdev)
- return -ENOMEM;
+ hdev = devm_drm_dev_alloc(&pdev->dev, &hl_driver, struct hl_device, drm);
+ if (IS_ERR(hdev))
+ return PTR_ERR(hdev);
+
+ hdev->dev = hdev->drm.dev;
/* Will be NULL in case of simulator device */
hdev->pdev = pdev;
/* Assign status description string */
- strncpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL], "operational", HL_STR_MAX);
- strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], "in reset", HL_STR_MAX);
- strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], "disabled", HL_STR_MAX);
- strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], "needs reset", HL_STR_MAX);
- strncpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
- "in device creation", HL_STR_MAX);
- strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE],
- "in reset after device release", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL], "operational", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], "in reset", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], "disabled", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], "needs reset", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
+ "in device creation", HL_STR_MAX);
+ strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE],
+ "in reset after device release", HL_STR_MAX);
/* First, we must find out which ASIC are we handling. This is needed
@@ -425,7 +476,7 @@ static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
if (hdev->asic_type == ASIC_INVALID) {
dev_err(&pdev->dev, "Unsupported ASIC\n");
rc = -ENODEV;
- goto free_hdev;
+ goto out_err;
}
copy_kernel_module_params_to_device(hdev);
@@ -434,42 +485,15 @@ static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
fixup_device_params(hdev);
- mutex_lock(&hl_devs_idr_lock);
-
- /* Always save 2 numbers, 1 for main device and 1 for control.
- * They must be consecutive
- */
- main_id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS, GFP_KERNEL);
-
- if (main_id >= 0)
- ctrl_id = idr_alloc(&hl_devs_idr, hdev, main_id + 1,
- main_id + 2, GFP_KERNEL);
-
- mutex_unlock(&hl_devs_idr_lock);
-
- if ((main_id < 0) || (ctrl_id < 0)) {
- if ((main_id == -ENOSPC) || (ctrl_id == -ENOSPC))
- pr_err("too many devices in the system\n");
-
- if (main_id >= 0) {
- mutex_lock(&hl_devs_idr_lock);
- idr_remove(&hl_devs_idr, main_id);
- mutex_unlock(&hl_devs_idr_lock);
- }
-
- rc = -EBUSY;
- goto free_hdev;
- }
-
- hdev->id = main_id;
- hdev->id_control = ctrl_id;
+ rc = allocate_device_id(hdev);
+ if (rc)
+ goto out_err;
*dev = hdev;
return 0;
-free_hdev:
- kfree(hdev);
+out_err:
return rc;
}
@@ -484,10 +508,8 @@ static void destroy_hdev(struct hl_device *hdev)
/* Remove device from the device list */
mutex_lock(&hl_devs_idr_lock);
idr_remove(&hl_devs_idr, hdev->id);
- idr_remove(&hl_devs_idr, hdev->id_control);
mutex_unlock(&hl_devs_idr_lock);
- kfree(hdev);
}
static int hl_pmops_suspend(struct device *dev)
@@ -691,28 +713,16 @@ static int __init hl_init(void)
hl_major = MAJOR(dev);
- hl_class = class_create(HL_NAME);
- if (IS_ERR(hl_class)) {
- pr_err("failed to allocate class\n");
- rc = PTR_ERR(hl_class);
- goto remove_major;
- }
-
- hl_debugfs_init();
-
rc = pci_register_driver(&hl_pci_driver);
if (rc) {
pr_err("failed to register pci device\n");
- goto remove_debugfs;
+ goto remove_major;
}
pr_debug("driver loaded\n");
return 0;
-remove_debugfs:
- hl_debugfs_fini();
- class_destroy(hl_class);
remove_major:
unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
return rc;
@@ -725,14 +735,6 @@ static void __exit hl_exit(void)
{
pci_unregister_driver(&hl_pci_driver);
- /*
- * Removing debugfs must be after all devices or simulator devices
- * have been removed because otherwise we get a bug in the
- * debugfs module for referencing NULL objects
- */
- hl_debugfs_fini();
-
- class_destroy(hl_class);
unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
idr_destroy(&hl_devs_idr);
diff --git a/drivers/accel/habanalabs/common/habanalabs_ioctl.c b/drivers/accel/habanalabs/common/habanalabs_ioctl.c
index 6a45a92344e9..8ef36effb95b 100644
--- a/drivers/accel/habanalabs/common/habanalabs_ioctl.c
+++ b/drivers/accel/habanalabs/common/habanalabs_ioctl.c
@@ -17,6 +17,8 @@
#include <linux/uaccess.h>
#include <linux/vmalloc.h>
+#include <asm/msr.h>
+
static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
[HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
[HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
@@ -320,6 +322,7 @@ static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
time_sync.host_time = ktime_get_raw_ns();
+ time_sync.tsc_time = rdtsc();
return copy_to_user(out, &time_sync,
min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
@@ -875,6 +878,28 @@ static int fw_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
return rc ? -EFAULT : 0;
}
+static int engine_err_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+ void __user *user_buf = (void __user *) (uintptr_t) args->return_pointer;
+ struct hl_device *hdev = hpriv->hdev;
+ u32 user_buf_size = args->return_size;
+ struct engine_err_info *info;
+ int rc;
+
+ if (!user_buf)
+ return -EINVAL;
+
+ info = &hdev->captured_err_info.engine_err;
+ if (!info->event_info_available)
+ return 0;
+
+ if (user_buf_size < sizeof(struct hl_info_engine_err_event))
+ return -ENOMEM;
+
+ rc = copy_to_user(user_buf, &info->event, sizeof(struct hl_info_engine_err_event));
+ return rc ? -EFAULT : 0;
+}
+
static int send_fw_generic_request(struct hl_device *hdev, struct hl_info_args *info_args)
{
void __user *buff = (void __user *) (uintptr_t) info_args->return_pointer;
@@ -1001,6 +1026,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
case HL_INFO_FW_ERR_EVENT:
return fw_err_info(hpriv, args);
+ case HL_INFO_USER_ENGINE_ERR_EVENT:
+ return engine_err_info(hpriv, args);
+
case HL_INFO_DRAM_USAGE:
return dram_usage_info(hpriv, args);
default:
@@ -1070,20 +1098,34 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
return rc;
}
-static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_info_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
+
return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
}
static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
{
+ struct hl_info_args *args = data;
+
+ switch (args->op) {
+ case HL_INFO_GET_EVENTS:
+ case HL_INFO_UNREGISTER_EVENTFD:
+ case HL_INFO_REGISTER_EVENTFD:
+ return -EOPNOTSUPP;
+ default:
+ break;
+ }
+
return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
}
-static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_debug_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
- struct hl_debug_args *args = data;
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
struct hl_device *hdev = hpriv->hdev;
+ struct hl_debug_args *args = data;
enum hl_device_status status;
int rc = 0;
@@ -1126,25 +1168,15 @@ static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
}
#define HL_IOCTL_DEF(ioctl, _func) \
- [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
-
-static const struct hl_ioctl_desc hl_ioctls[] = {
- HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_wait_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
-};
+ [_IOC_NR(ioctl) - HL_COMMAND_START] = {.cmd = ioctl, .func = _func}
static const struct hl_ioctl_desc hl_ioctls_control[] = {
- HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
+ HL_IOCTL_DEF(DRM_IOCTL_HL_INFO, hl_info_ioctl_control)
};
-static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
- const struct hl_ioctl_desc *ioctl, struct device *dev)
+static long _hl_ioctl(struct hl_fpriv *hpriv, unsigned int cmd, unsigned long arg,
+ const struct hl_ioctl_desc *ioctl, struct device *dev)
{
- struct hl_fpriv *hpriv = filep->private_data;
unsigned int nr = _IOC_NR(cmd);
char stack_kdata[128] = {0};
char *kdata = NULL;
@@ -1194,9 +1226,13 @@ static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
retcode = -EFAULT;
out_err:
- if (retcode)
- dev_dbg_ratelimited(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
- task_pid_nr(current), cmd, nr);
+ if (retcode) {
+ char task_comm[TASK_COMM_LEN];
+
+ dev_dbg_ratelimited(dev,
+ "error in ioctl: pid=%d, comm=\"%s\", cmd=%#010x, nr=%#04x\n",
+ task_pid_nr(current), get_task_comm(task_comm, current), cmd, nr);
+ }
if (kdata != stack_kdata)
kfree(kdata);
@@ -1204,29 +1240,6 @@ out_err:
return retcode;
}
-long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
-{
- struct hl_fpriv *hpriv = filep->private_data;
- struct hl_device *hdev = hpriv->hdev;
- const struct hl_ioctl_desc *ioctl = NULL;
- unsigned int nr = _IOC_NR(cmd);
-
- if (!hdev) {
- pr_err_ratelimited("Sending ioctl after device was removed! Please close FD\n");
- return -ENODEV;
- }
-
- if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
- ioctl = &hl_ioctls[nr];
- } else {
- dev_dbg_ratelimited(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
- task_pid_nr(current), nr);
- return -ENOTTY;
- }
-
- return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
-}
-
long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
{
struct hl_fpriv *hpriv = filep->private_data;
@@ -1239,13 +1252,16 @@ long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
return -ENODEV;
}
- if (nr == _IOC_NR(HL_IOCTL_INFO)) {
- ioctl = &hl_ioctls_control[nr];
+ if (nr == _IOC_NR(DRM_IOCTL_HL_INFO)) {
+ ioctl = &hl_ioctls_control[nr - HL_COMMAND_START];
} else {
- dev_dbg_ratelimited(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
- task_pid_nr(current), nr);
+ char task_comm[TASK_COMM_LEN];
+
+ dev_dbg_ratelimited(hdev->dev_ctrl,
+ "invalid ioctl: pid=%d, comm=\"%s\", cmd=%#010x, nr=%#04x\n",
+ task_pid_nr(current), get_task_comm(task_comm, current), cmd, nr);
return -ENOTTY;
}
- return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
+ return _hl_ioctl(hpriv, cmd, arg, ioctl, hdev->dev_ctrl);
}
diff --git a/drivers/accel/habanalabs/common/irq.c b/drivers/accel/habanalabs/common/irq.c
index b1010d206c2e..978b7f4d5eeb 100644
--- a/drivers/accel/habanalabs/common/irq.c
+++ b/drivers/accel/habanalabs/common/irq.c
@@ -204,8 +204,10 @@ static void hl_ts_free_objects(struct work_struct *work)
{
struct timestamp_reg_work_obj *job =
container_of(work, struct timestamp_reg_work_obj, free_obj);
+ struct list_head *dynamic_alloc_free_list_head = job->dynamic_alloc_free_obj_head;
struct timestamp_reg_free_node *free_obj, *temp_free_obj;
struct list_head *free_list_head = job->free_obj_head;
+
struct hl_device *hdev = job->hdev;
list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
@@ -215,10 +217,28 @@ static void hl_ts_free_objects(struct work_struct *work)
hl_mmap_mem_buf_put(free_obj->buf);
hl_cb_put(free_obj->cq_cb);
- kfree(free_obj);
+ atomic_set(&free_obj->in_use, 0);
}
kfree(free_list_head);
+
+ if (dynamic_alloc_free_list_head) {
+ list_for_each_entry_safe(free_obj, temp_free_obj, dynamic_alloc_free_list_head,
+ free_objects_node) {
+ dev_dbg(hdev->dev,
+ "Dynamic_Alloc list: About to put refcount to buf (%p) cq_cb(%p)\n",
+ free_obj->buf,
+ free_obj->cq_cb);
+
+ hl_mmap_mem_buf_put(free_obj->buf);
+ hl_cb_put(free_obj->cq_cb);
+ list_del(&free_obj->free_objects_node);
+ kfree(free_obj);
+ }
+
+ kfree(dynamic_alloc_free_list_head);
+ }
+
kfree(job);
}
@@ -233,11 +253,18 @@ static void hl_ts_free_objects(struct work_struct *work)
* list to a dedicated workqueue to do the actual put.
*/
static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
- struct list_head **free_list, ktime_t now)
+ struct list_head **free_list,
+ struct list_head **dynamic_alloc_list,
+ struct hl_user_interrupt *intr)
{
+ struct hl_ts_free_jobs *ts_free_jobs_data;
struct timestamp_reg_free_node *free_node;
+ u32 free_node_index;
u64 timestamp;
+ ts_free_jobs_data = &intr->ts_free_jobs_data;
+ free_node_index = ts_free_jobs_data->next_avail_free_node_idx;
+
if (!(*free_list)) {
/* Alloc/Init the timestamp registration free objects list */
*free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
@@ -247,39 +274,65 @@ static int handle_registration_node(struct hl_device *hdev, struct hl_user_pendi
INIT_LIST_HEAD(*free_list);
}
- free_node = kmalloc(sizeof(*free_node), GFP_ATOMIC);
- if (!free_node)
- return -ENOMEM;
+ free_node = &ts_free_jobs_data->free_nodes_pool[free_node_index];
+ if (atomic_cmpxchg(&free_node->in_use, 0, 1)) {
+ dev_dbg(hdev->dev,
+ "Timestamp free node pool is full, buff: %p, record: %p, irq: %u\n",
+ pend->ts_reg_info.buf,
+ pend,
+ intr->interrupt_id);
- timestamp = ktime_to_ns(now);
+ if (!(*dynamic_alloc_list)) {
+ *dynamic_alloc_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
+ if (!(*dynamic_alloc_list))
+ return -ENOMEM;
- *pend->ts_reg_info.timestamp_kernel_addr = timestamp;
+ INIT_LIST_HEAD(*dynamic_alloc_list);
+ }
+
+ free_node = kmalloc(sizeof(struct timestamp_reg_free_node), GFP_ATOMIC);
+ if (!free_node)
+ return -ENOMEM;
+
+ free_node->dynamic_alloc = 1;
+ }
- dev_dbg(hdev->dev, "Timestamp is set to ts cb address (%p), ts: 0x%llx\n",
- pend->ts_reg_info.timestamp_kernel_addr,
- *(u64 *)pend->ts_reg_info.timestamp_kernel_addr);
+ timestamp = ktime_to_ns(intr->timestamp);
- list_del(&pend->wait_list_node);
+ *pend->ts_reg_info.timestamp_kernel_addr = timestamp;
+
+ dev_dbg(hdev->dev, "Irq handle: Timestamp record (%p) ts cb address (%p), interrupt_id: %u\n",
+ pend, pend->ts_reg_info.timestamp_kernel_addr, intr->interrupt_id);
- /* Mark kernel CB node as free */
- pend->ts_reg_info.in_use = 0;
+ list_del(&pend->list_node);
/* Putting the refcount for ts_buff and cq_cb objects will be handled
* in workqueue context, just add job to free_list.
*/
free_node->buf = pend->ts_reg_info.buf;
free_node->cq_cb = pend->ts_reg_info.cq_cb;
- list_add(&free_node->free_objects_node, *free_list);
+
+ if (free_node->dynamic_alloc) {
+ list_add(&free_node->free_objects_node, *dynamic_alloc_list);
+ } else {
+ ts_free_jobs_data->next_avail_free_node_idx =
+ (++free_node_index) % ts_free_jobs_data->free_nodes_length;
+ list_add(&free_node->free_objects_node, *free_list);
+ }
+
+ /* Mark TS record as free */
+ pend->ts_reg_info.in_use = false;
return 0;
}
-static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interrupt *intr)
+static void handle_user_interrupt_ts_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
{
+ struct list_head *ts_reg_free_list_head = NULL, *dynamic_alloc_list_head = NULL;
struct hl_user_pending_interrupt *pend, *temp_pend;
- struct list_head *ts_reg_free_list_head = NULL;
struct timestamp_reg_work_obj *job;
bool reg_node_handle_fail = false;
+ unsigned long flags;
int rc;
/* For registration nodes:
@@ -288,36 +341,32 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
* or in irq handler context at all (since release functions are long and
* might sleep), so we will need to handle that part in workqueue context.
* To avoid handling kmalloc failure which compels us rolling back actions
- * and move nodes hanged on the free list back to the interrupt wait list
+ * and move nodes hanged on the free list back to the interrupt ts list
* we always alloc the job of the WQ at the beginning.
*/
job = kmalloc(sizeof(*job), GFP_ATOMIC);
if (!job)
return;
- spin_lock(&intr->wait_list_lock);
- list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, wait_list_node) {
+ spin_lock_irqsave(&intr->ts_list_lock, flags);
+ list_for_each_entry_safe(pend, temp_pend, &intr->ts_list_head, list_node) {
if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
!pend->cq_kernel_addr) {
- if (pend->ts_reg_info.buf) {
- if (!reg_node_handle_fail) {
- rc = handle_registration_node(hdev, pend,
- &ts_reg_free_list_head, intr->timestamp);
- if (rc)
- reg_node_handle_fail = true;
- }
- } else {
- /* Handle wait target value node */
- pend->fence.timestamp = intr->timestamp;
- complete_all(&pend->fence.completion);
+ if (!reg_node_handle_fail) {
+ rc = handle_registration_node(hdev, pend,
+ &ts_reg_free_list_head,
+ &dynamic_alloc_list_head, intr);
+ if (rc)
+ reg_node_handle_fail = true;
}
}
}
- spin_unlock(&intr->wait_list_lock);
+ spin_unlock_irqrestore(&intr->ts_list_lock, flags);
if (ts_reg_free_list_head) {
INIT_WORK(&job->free_obj, hl_ts_free_objects);
job->free_obj_head = ts_reg_free_list_head;
+ job->dynamic_alloc_free_obj_head = dynamic_alloc_list_head;
job->hdev = hdev;
queue_work(hdev->ts_free_obj_wq, &job->free_obj);
} else {
@@ -325,6 +374,23 @@ static void handle_user_interrupt(struct hl_device *hdev, struct hl_user_interru
}
}
+static void handle_user_interrupt_wait_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
+{
+ struct hl_user_pending_interrupt *pend, *temp_pend;
+ unsigned long flags;
+
+ spin_lock_irqsave(&intr->wait_list_lock, flags);
+ list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, list_node) {
+ if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
+ !pend->cq_kernel_addr) {
+ /* Handle wait target value node */
+ pend->fence.timestamp = intr->timestamp;
+ complete_all(&pend->fence.completion);
+ }
+ }
+ spin_unlock_irqrestore(&intr->wait_list_lock, flags);
+}
+
static void handle_tpc_interrupt(struct hl_device *hdev)
{
u64 event_mask;
@@ -346,19 +412,38 @@ static void handle_unexpected_user_interrupt(struct hl_device *hdev)
}
/**
- * hl_irq_handler_user_interrupt - irq handler for user interrupts
+ * hl_irq_user_interrupt_handler - irq handler for user interrupts.
*
* @irq: irq number
* @arg: pointer to user interrupt structure
- *
*/
-irqreturn_t hl_irq_handler_user_interrupt(int irq, void *arg)
+irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg)
{
struct hl_user_interrupt *user_int = arg;
+ struct hl_device *hdev = user_int->hdev;
user_int->timestamp = ktime_get();
+ switch (user_int->type) {
+ case HL_USR_INTERRUPT_CQ:
+ /* First handle user waiters threads */
+ handle_user_interrupt_wait_list(hdev, &hdev->common_user_cq_interrupt);
+ handle_user_interrupt_wait_list(hdev, user_int);
- return IRQ_WAKE_THREAD;
+ /* Second handle user timestamp registrations */
+ handle_user_interrupt_ts_list(hdev, &hdev->common_user_cq_interrupt);
+ handle_user_interrupt_ts_list(hdev, user_int);
+ break;
+ case HL_USR_INTERRUPT_DECODER:
+ handle_user_interrupt_wait_list(hdev, &hdev->common_decoder_interrupt);
+
+ /* Handle decoder interrupt registered on this specific irq */
+ handle_user_interrupt_wait_list(hdev, user_int);
+ break;
+ default:
+ break;
+ }
+
+ return IRQ_HANDLED;
}
/**
@@ -374,19 +459,8 @@ irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
struct hl_user_interrupt *user_int = arg;
struct hl_device *hdev = user_int->hdev;
+ user_int->timestamp = ktime_get();
switch (user_int->type) {
- case HL_USR_INTERRUPT_CQ:
- handle_user_interrupt(hdev, &hdev->common_user_cq_interrupt);
-
- /* Handle user cq interrupt registered on this specific irq */
- handle_user_interrupt(hdev, user_int);
- break;
- case HL_USR_INTERRUPT_DECODER:
- handle_user_interrupt(hdev, &hdev->common_decoder_interrupt);
-
- /* Handle decoder interrupt registered on this specific irq */
- handle_user_interrupt(hdev, user_int);
- break;
case HL_USR_INTERRUPT_TPC:
handle_tpc_interrupt(hdev);
break;
@@ -400,6 +474,18 @@ irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
return IRQ_HANDLED;
}
+irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg)
+{
+ u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
+ struct hl_device *hdev = arg;
+
+ dev_err(hdev->dev, "EQ error interrupt received\n");
+
+ hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
+
+ return IRQ_HANDLED;
+}
+
/**
* hl_irq_handler_eq - irq handler for event queue
*
diff --git a/drivers/accel/habanalabs/common/memory.c b/drivers/accel/habanalabs/common/memory.c
index 4fc72a07d2f5..0b8689fe0b64 100644
--- a/drivers/accel/habanalabs/common/memory.c
+++ b/drivers/accel/habanalabs/common/memory.c
@@ -244,7 +244,7 @@ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size,
*p_userptr = userptr;
- rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, DMA_BIDIRECTIONAL);
+ rc = hl_dma_map_sgtable(hdev, userptr->sgt, DMA_BIDIRECTIONAL);
if (rc) {
dev_err(hdev->dev, "failed to map sgt with DMA region\n");
goto dma_map_err;
@@ -832,7 +832,6 @@ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
* physical pages
*
* This function does the following:
- * - Pin the physical pages related to the given virtual block.
* - Create a physical page pack from the physical pages related to the given
* virtual block.
*/
@@ -1532,24 +1531,20 @@ static int set_dma_sg(struct scatterlist *sg, u64 bar_address, u64 chunk_size,
}
static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 *pages, u64 npages,
- u64 page_size, u64 exported_size,
+ u64 page_size, u64 exported_size, u64 offset,
struct device *dev, enum dma_data_direction dir)
{
- u64 chunk_size, bar_address, dma_max_seg_size, cur_size_to_export, cur_npages;
- struct asic_fixed_properties *prop;
- int rc, i, j, nents, cur_page;
+ u64 dma_max_seg_size, curr_page, size, chunk_size, left_size_to_export, left_size_in_page,
+ left_size_in_dma_seg, device_address, bar_address, start_page;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
struct scatterlist *sg;
+ unsigned int nents, i;
struct sg_table *sgt;
+ bool next_sg_entry;
+ int rc;
- prop = &hdev->asic_prop;
-
- dma_max_seg_size = dma_get_max_seg_size(dev);
-
- /* We would like to align the max segment size to PAGE_SIZE, so the
- * SGL will contain aligned addresses that can be easily mapped to
- * an MMU
- */
- dma_max_seg_size = ALIGN_DOWN(dma_max_seg_size, PAGE_SIZE);
+ /* Align max segment size to PAGE_SIZE to fit the minimal IOMMU mapping granularity */
+ dma_max_seg_size = ALIGN_DOWN(dma_get_max_seg_size(dev), PAGE_SIZE);
if (dma_max_seg_size < PAGE_SIZE) {
dev_err_ratelimited(hdev->dev,
"dma_max_seg_size %llu can't be smaller than PAGE_SIZE\n",
@@ -1561,121 +1556,149 @@ static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64
if (!sgt)
return ERR_PTR(-ENOMEM);
- /* remove export size restrictions in case not explicitly defined */
- cur_size_to_export = exported_size ? exported_size : (npages * page_size);
-
- /* If the size of each page is larger than the dma max segment size,
- * then we can't combine pages and the number of entries in the SGL
- * will just be the
- * <number of pages> * <chunks of max segment size in each page>
- */
- if (page_size > dma_max_seg_size) {
- /* we should limit number of pages according to the exported size */
- cur_npages = DIV_ROUND_UP_SECTOR_T(cur_size_to_export, page_size);
- nents = cur_npages * DIV_ROUND_UP_SECTOR_T(page_size, dma_max_seg_size);
- } else {
- cur_npages = npages;
-
- /* Get number of non-contiguous chunks */
- for (i = 1, nents = 1, chunk_size = page_size ; i < cur_npages ; i++) {
- if (pages[i - 1] + page_size != pages[i] ||
- chunk_size + page_size > dma_max_seg_size) {
- nents++;
- chunk_size = page_size;
- continue;
- }
+ /* Use the offset to move to the actual first page that is exported */
+ for (start_page = 0 ; start_page < npages ; ++start_page) {
+ if (offset < page_size)
+ break;
- chunk_size += page_size;
- }
+ /* The offset value was validated so there can't be an underflow */
+ offset -= page_size;
}
- rc = sg_alloc_table(sgt, nents, GFP_KERNEL | __GFP_ZERO);
- if (rc)
- goto error_free;
+ /* Calculate the required number of entries for the SG table */
+ curr_page = start_page;
+ nents = 1;
+ left_size_to_export = exported_size;
+ left_size_in_page = page_size - offset;
+ left_size_in_dma_seg = dma_max_seg_size;
+ next_sg_entry = false;
- cur_page = 0;
+ while (true) {
+ size = min3(left_size_to_export, left_size_in_page, left_size_in_dma_seg);
+ left_size_to_export -= size;
+ left_size_in_page -= size;
+ left_size_in_dma_seg -= size;
- if (page_size > dma_max_seg_size) {
- u64 size_left, cur_device_address = 0;
+ if (!left_size_to_export)
+ break;
- size_left = page_size;
+ if (!left_size_in_page) {
+ /* left_size_to_export is not zero so there must be another page */
+ if (pages[curr_page] + page_size != pages[curr_page + 1])
+ next_sg_entry = true;
- /* Need to split each page into the number of chunks of
- * dma_max_seg_size
- */
- for_each_sgtable_dma_sg(sgt, sg, i) {
- if (size_left == page_size)
- cur_device_address =
- pages[cur_page] - prop->dram_base_address;
- else
- cur_device_address += dma_max_seg_size;
+ ++curr_page;
+ left_size_in_page = page_size;
+ }
- /* make sure not to export over exported size */
- chunk_size = min3(size_left, dma_max_seg_size, cur_size_to_export);
+ if (!left_size_in_dma_seg) {
+ next_sg_entry = true;
+ left_size_in_dma_seg = dma_max_seg_size;
+ }
- bar_address = hdev->dram_pci_bar_start + cur_device_address;
+ if (next_sg_entry) {
+ ++nents;
+ next_sg_entry = false;
+ }
+ }
- rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir);
- if (rc)
- goto error_unmap;
+ rc = sg_alloc_table(sgt, nents, GFP_KERNEL | __GFP_ZERO);
+ if (rc)
+ goto err_free_sgt;
- cur_size_to_export -= chunk_size;
+ /* Prepare the SG table entries */
+ curr_page = start_page;
+ device_address = pages[curr_page] + offset;
+ left_size_to_export = exported_size;
+ left_size_in_page = page_size - offset;
+ left_size_in_dma_seg = dma_max_seg_size;
+ next_sg_entry = false;
- if (size_left > dma_max_seg_size) {
- size_left -= dma_max_seg_size;
- } else {
- cur_page++;
- size_left = page_size;
+ for_each_sgtable_dma_sg(sgt, sg, i) {
+ bar_address = hdev->dram_pci_bar_start + (device_address - prop->dram_base_address);
+ chunk_size = 0;
+
+ for ( ; curr_page < npages ; ++curr_page) {
+ size = min3(left_size_to_export, left_size_in_page, left_size_in_dma_seg);
+ chunk_size += size;
+ left_size_to_export -= size;
+ left_size_in_page -= size;
+ left_size_in_dma_seg -= size;
+
+ if (!left_size_to_export)
+ break;
+
+ if (!left_size_in_page) {
+ /* left_size_to_export is not zero so there must be another page */
+ if (pages[curr_page] + page_size != pages[curr_page + 1]) {
+ device_address = pages[curr_page + 1];
+ next_sg_entry = true;
+ }
+
+ left_size_in_page = page_size;
}
- }
- } else {
- /* Merge pages and put them into the scatterlist */
- for_each_sgtable_dma_sg(sgt, sg, i) {
- chunk_size = page_size;
- for (j = cur_page + 1 ; j < cur_npages ; j++) {
- if (pages[j - 1] + page_size != pages[j] ||
- chunk_size + page_size > dma_max_seg_size)
- break;
-
- chunk_size += page_size;
+
+ if (!left_size_in_dma_seg) {
+ /*
+ * Skip setting a new device address if already moving to a page
+ * which is not contiguous with the current page.
+ */
+ if (!next_sg_entry) {
+ device_address += chunk_size;
+ next_sg_entry = true;
+ }
+
+ left_size_in_dma_seg = dma_max_seg_size;
}
- bar_address = hdev->dram_pci_bar_start +
- (pages[cur_page] - prop->dram_base_address);
+ if (next_sg_entry) {
+ next_sg_entry = false;
+ break;
+ }
+ }
- /* make sure not to export over exported size */
- chunk_size = min(chunk_size, cur_size_to_export);
- rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir);
- if (rc)
- goto error_unmap;
+ rc = set_dma_sg(sg, bar_address, chunk_size, dev, dir);
+ if (rc)
+ goto err_unmap;
+ }
- cur_size_to_export -= chunk_size;
- cur_page = j;
- }
+ /* There should be nothing left to export exactly after looping over all SG elements */
+ if (left_size_to_export) {
+ dev_err(hdev->dev,
+ "left size to export %#llx after initializing %u SG elements\n",
+ left_size_to_export, sgt->nents);
+ rc = -ENOMEM;
+ goto err_unmap;
}
- /* Because we are not going to include a CPU list we want to have some
- * chance that other users will detect this by setting the orig_nents
- * to 0 and using only nents (length of DMA list) when going over the
- * sgl
+ /*
+ * Because we are not going to include a CPU list, we want to have some chance that other
+ * users will detect this when going over SG table, by setting the orig_nents to 0 and using
+ * only nents (length of DMA list).
*/
sgt->orig_nents = 0;
+ dev_dbg(hdev->dev, "prepared SG table with %u entries for importer %s\n",
+ nents, dev_name(dev));
+ for_each_sgtable_dma_sg(sgt, sg, i)
+ dev_dbg(hdev->dev,
+ "SG entry %d: address %#llx, length %#x\n",
+ i, sg_dma_address(sg), sg_dma_len(sg));
+
return sgt;
-error_unmap:
+err_unmap:
for_each_sgtable_dma_sg(sgt, sg, i) {
if (!sg_dma_len(sg))
continue;
- dma_unmap_resource(dev, sg_dma_address(sg),
- sg_dma_len(sg), dir,
+ dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg), dir,
DMA_ATTR_SKIP_CPU_SYNC);
}
sg_free_table(sgt);
-error_free:
+err_free_sgt:
kfree(sgt);
return ERR_PTR(rc);
}
@@ -1700,6 +1723,7 @@ static int hl_dmabuf_attach(struct dma_buf *dmabuf,
static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment,
enum dma_data_direction dir)
{
+ u64 *pages, npages, page_size, exported_size, offset;
struct dma_buf *dma_buf = attachment->dmabuf;
struct hl_vm_phys_pg_pack *phys_pg_pack;
struct hl_dmabuf_priv *hl_dmabuf;
@@ -1708,30 +1732,28 @@ static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment,
hl_dmabuf = dma_buf->priv;
hdev = hl_dmabuf->ctx->hdev;
- phys_pg_pack = hl_dmabuf->phys_pg_pack;
if (!attachment->peer2peer) {
dev_dbg(hdev->dev, "Failed to map dmabuf because p2p is disabled\n");
return ERR_PTR(-EPERM);
}
- if (phys_pg_pack)
- sgt = alloc_sgt_from_device_pages(hdev,
- phys_pg_pack->pages,
- phys_pg_pack->npages,
- phys_pg_pack->page_size,
- phys_pg_pack->exported_size,
- attachment->dev,
- dir);
- else
- sgt = alloc_sgt_from_device_pages(hdev,
- &hl_dmabuf->device_address,
- 1,
- hl_dmabuf->dmabuf->size,
- 0,
- attachment->dev,
- dir);
+ exported_size = hl_dmabuf->dmabuf->size;
+ offset = hl_dmabuf->offset;
+ phys_pg_pack = hl_dmabuf->phys_pg_pack;
+ if (phys_pg_pack) {
+ pages = phys_pg_pack->pages;
+ npages = phys_pg_pack->npages;
+ page_size = phys_pg_pack->page_size;
+ } else {
+ pages = &hl_dmabuf->device_phys_addr;
+ npages = 1;
+ page_size = hl_dmabuf->dmabuf->size;
+ }
+
+ sgt = alloc_sgt_from_device_pages(hdev, pages, npages, page_size, exported_size, offset,
+ attachment->dev, dir);
if (IS_ERR(sgt))
dev_err(hdev->dev, "failed (%ld) to initialize sgt for dmabuf\n", PTR_ERR(sgt));
@@ -1818,7 +1840,7 @@ static void hl_release_dmabuf(struct dma_buf *dmabuf)
hl_ctx_put(ctx);
/* Paired with get_file() in export_dmabuf() */
- fput(ctx->hpriv->filp);
+ fput(ctx->hpriv->file_priv->filp);
kfree(hl_dmabuf);
}
@@ -1864,7 +1886,7 @@ static int export_dmabuf(struct hl_ctx *ctx,
* released first and only then the compute device.
* Paired with fput() in hl_release_dmabuf().
*/
- get_file(ctx->hpriv->filp);
+ get_file(ctx->hpriv->file_priv->filp);
*dmabuf_fd = fd;
@@ -1876,22 +1898,29 @@ err_dma_buf_put:
return rc;
}
-static int validate_export_params_common(struct hl_device *hdev, u64 device_addr, u64 size)
+static int validate_export_params_common(struct hl_device *hdev, u64 addr, u64 size, u64 offset)
{
- if (!IS_ALIGNED(device_addr, PAGE_SIZE)) {
+ if (!PAGE_ALIGNED(addr)) {
dev_dbg(hdev->dev,
- "exported device memory address 0x%llx should be aligned to 0x%lx\n",
- device_addr, PAGE_SIZE);
+ "exported device memory address 0x%llx should be aligned to PAGE_SIZE 0x%lx\n",
+ addr, PAGE_SIZE);
return -EINVAL;
}
- if (size < PAGE_SIZE) {
+ if (!size || !PAGE_ALIGNED(size)) {
dev_dbg(hdev->dev,
- "exported device memory size %llu should be equal to or greater than %lu\n",
+ "exported device memory size %llu should be a multiple of PAGE_SIZE %lu\n",
size, PAGE_SIZE);
return -EINVAL;
}
+ if (!PAGE_ALIGNED(offset)) {
+ dev_dbg(hdev->dev,
+ "exported device memory offset %llu should be a multiple of PAGE_SIZE %lu\n",
+ offset, PAGE_SIZE);
+ return -EINVAL;
+ }
+
return 0;
}
@@ -1901,13 +1930,13 @@ static int validate_export_params_no_mmu(struct hl_device *hdev, u64 device_addr
u64 bar_address;
int rc;
- rc = validate_export_params_common(hdev, device_addr, size);
+ rc = validate_export_params_common(hdev, device_addr, size, 0);
if (rc)
return rc;
if (device_addr < prop->dram_user_base_address ||
- (device_addr + size) > prop->dram_end_address ||
- (device_addr + size) < device_addr) {
+ (device_addr + size) > prop->dram_end_address ||
+ (device_addr + size) < device_addr) {
dev_dbg(hdev->dev,
"DRAM memory range 0x%llx (+0x%llx) is outside of DRAM boundaries\n",
device_addr, size);
@@ -1934,29 +1963,26 @@ static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 s
u64 bar_address;
int i, rc;
- rc = validate_export_params_common(hdev, device_addr, size);
+ rc = validate_export_params_common(hdev, device_addr, size, offset);
if (rc)
return rc;
if ((offset + size) > phys_pg_pack->total_size) {
dev_dbg(hdev->dev, "offset %#llx and size %#llx exceed total map size %#llx\n",
- offset, size, phys_pg_pack->total_size);
+ offset, size, phys_pg_pack->total_size);
return -EINVAL;
}
for (i = 0 ; i < phys_pg_pack->npages ; i++) {
-
bar_address = hdev->dram_pci_bar_start +
- (phys_pg_pack->pages[i] - prop->dram_base_address);
+ (phys_pg_pack->pages[i] - prop->dram_base_address);
if ((bar_address + phys_pg_pack->page_size) >
(hdev->dram_pci_bar_start + prop->dram_pci_bar_size) ||
(bar_address + phys_pg_pack->page_size) < bar_address) {
dev_dbg(hdev->dev,
"DRAM memory range 0x%llx (+0x%x) is outside of PCI BAR boundaries\n",
- phys_pg_pack->pages[i],
- phys_pg_pack->page_size);
-
+ phys_pg_pack->pages[i], phys_pg_pack->page_size);
return -EINVAL;
}
}
@@ -2012,7 +2038,6 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o
struct asic_fixed_properties *prop;
struct hl_dmabuf_priv *hl_dmabuf;
struct hl_device *hdev;
- u64 export_addr;
int rc;
hdev = ctx->hdev;
@@ -2024,8 +2049,6 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o
return -EINVAL;
}
- export_addr = addr + offset;
-
hl_dmabuf = kzalloc(sizeof(*hl_dmabuf), GFP_KERNEL);
if (!hl_dmabuf)
return -ENOMEM;
@@ -2041,20 +2064,20 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o
rc = PTR_ERR(phys_pg_pack);
goto dec_memhash_export_cnt;
}
- rc = validate_export_params(hdev, export_addr, size, offset, phys_pg_pack);
+ rc = validate_export_params(hdev, addr, size, offset, phys_pg_pack);
if (rc)
goto dec_memhash_export_cnt;
- phys_pg_pack->exported_size = size;
hl_dmabuf->phys_pg_pack = phys_pg_pack;
hl_dmabuf->memhash_hnode = hnode;
+ hl_dmabuf->offset = offset;
} else {
- rc = validate_export_params_no_mmu(hdev, export_addr, size);
+ rc = validate_export_params_no_mmu(hdev, addr, size);
if (rc)
goto err_free_dmabuf_wrapper;
- }
- hl_dmabuf->device_address = export_addr;
+ hl_dmabuf->device_phys_addr = addr;
+ }
rc = export_dmabuf(ctx, hl_dmabuf, size, flags, dmabuf_fd);
if (rc)
@@ -2171,8 +2194,9 @@ static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in
return 0;
}
-int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
+int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv)
{
+ struct hl_fpriv *hpriv = file_priv->driver_priv;
enum hl_device_status status;
union hl_mem_args *args = data;
struct hl_device *hdev = hpriv->hdev;
@@ -2420,7 +2444,7 @@ void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
hl_debugfs_remove_userptr(hdev, userptr);
if (userptr->dma_mapped)
- hdev->asic_funcs->hl_dma_unmap_sgtable(hdev, userptr->sgt, userptr->dir);
+ hl_dma_unmap_sgtable(hdev, userptr->sgt, userptr->dir);
unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true);
kvfree(userptr->pages);
diff --git a/drivers/accel/habanalabs/gaudi/gaudi.c b/drivers/accel/habanalabs/gaudi/gaudi.c
index 056e2ef44afb..53292d4c15c8 100644
--- a/drivers/accel/habanalabs/gaudi/gaudi.c
+++ b/drivers/accel/habanalabs/gaudi/gaudi.c
@@ -63,6 +63,10 @@
#define GAUDI_LINUX_FW_FILE "habanalabs/gaudi/gaudi-fit.itb"
#define GAUDI_TPC_FW_FILE "habanalabs/gaudi/gaudi_tpc.bin"
+MODULE_FIRMWARE(GAUDI_BOOT_FIT_FILE);
+MODULE_FIRMWARE(GAUDI_LINUX_FW_FILE);
+MODULE_FIRMWARE(GAUDI_TPC_FW_FILE);
+
#define GAUDI_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */
#define GAUDI_RESET_TIMEOUT_MSEC 2000 /* 2000ms */
@@ -660,7 +664,7 @@ static int gaudi_set_fixed_properties(struct hl_device *hdev)
prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
- strncpy(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+ strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
CARD_NAME_MAX_LEN);
prop->max_pending_cs = GAUDI_MAX_PENDING_CS;
@@ -4619,8 +4623,7 @@ static int gaudi_scrub_device_dram(struct hl_device *hdev, u64 val)
static int gaudi_scrub_device_mem(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
- u64 wait_to_idle_time = hdev->pdev ? HBM_SCRUBBING_TIMEOUT_US :
- min_t(u64, HBM_SCRUBBING_TIMEOUT_US * 10, HL_SIM_MAX_TIMEOUT_US);
+ u64 wait_to_idle_time = HBM_SCRUBBING_TIMEOUT_US;
u64 addr, size, val = hdev->memory_scrub_val;
ktime_t timeout;
int rc = 0;
@@ -4904,7 +4907,7 @@ static int gaudi_pin_memory_before_cs(struct hl_device *hdev,
list_add_tail(&userptr->job_node, parser->job_userptr_list);
- rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, dir);
+ rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir);
if (rc) {
dev_err(hdev->dev, "failed to map sgt with DMA region\n");
goto unpin_memory;
@@ -8000,7 +8003,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev)
return rc;
if (!strlen(prop->cpucp_info.card_name))
- strncpy(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+ strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
CARD_NAME_MAX_LEN);
hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);
@@ -9140,9 +9143,9 @@ static const struct hl_asic_funcs gaudi_funcs = {
.asic_dma_pool_free = gaudi_dma_pool_free,
.cpu_accessible_dma_pool_alloc = gaudi_cpu_accessible_dma_pool_alloc,
.cpu_accessible_dma_pool_free = gaudi_cpu_accessible_dma_pool_free,
- .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
+ .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable,
.cs_parser = gaudi_cs_parser,
- .asic_dma_map_sgtable = hl_dma_map_sgtable,
+ .dma_map_sgtable = hl_asic_dma_map_sgtable,
.add_end_of_cb_packets = gaudi_add_end_of_cb_packets,
.update_eq_ci = gaudi_update_eq_ci,
.context_switch = gaudi_context_switch,
diff --git a/drivers/accel/habanalabs/gaudi/gaudiP.h b/drivers/accel/habanalabs/gaudi/gaudiP.h
index b8fa724be5a1..831be53bb9d7 100644
--- a/drivers/accel/habanalabs/gaudi/gaudiP.h
+++ b/drivers/accel/habanalabs/gaudi/gaudiP.h
@@ -10,7 +10,7 @@
#include <uapi/drm/habanalabs_accel.h>
#include "../common/habanalabs.h"
-#include "../include/common/hl_boot_if.h"
+#include <linux/habanalabs/hl_boot_if.h>
#include "../include/gaudi/gaudi_packets.h"
#include "../include/gaudi/gaudi.h"
#include "../include/gaudi/gaudi_async_events.h"
diff --git a/drivers/accel/habanalabs/gaudi/gaudi_coresight.c b/drivers/accel/habanalabs/gaudi/gaudi_coresight.c
index 3455b14554c6..1168fefa33f4 100644
--- a/drivers/accel/habanalabs/gaudi/gaudi_coresight.c
+++ b/drivers/accel/habanalabs/gaudi/gaudi_coresight.c
@@ -482,6 +482,11 @@ static int gaudi_config_etf(struct hl_device *hdev,
WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+ val = RREG32(base_reg + 0x20);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
val = RREG32(base_reg + 0x304);
val |= 0x1000;
WREG32(base_reg + 0x304, val);
@@ -580,6 +585,13 @@ static int gaudi_config_etr(struct hl_device *hdev,
WREG32(mmPSOC_ETR_LAR, CORESIGHT_UNLOCK);
+ val = RREG32(mmPSOC_ETR_CTL);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
+
+
val = RREG32(mmPSOC_ETR_FFCR);
val |= 0x1000;
WREG32(mmPSOC_ETR_FFCR, val);
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2.c b/drivers/accel/habanalabs/gaudi2/gaudi2.c
index 20c4583f12b0..819660c684cf 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2.c
@@ -66,7 +66,6 @@
#define GAUDI2_NUM_OF_TPC_INTR_CAUSE 31
#define GAUDI2_NUM_OF_DEC_ERR_CAUSE 25
#define GAUDI2_NUM_OF_MME_ERR_CAUSE 16
-#define GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE 5
#define GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE 7
#define GAUDI2_NUM_OF_DMA_CORE_INTR_CAUSE 8
#define GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE 19
@@ -916,14 +915,6 @@ static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] =
"sbte_prtn_intr_4",
};
-static const char * const guadi2_mme_sbte_error_cause[GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE] = {
- "i0",
- "i1",
- "i2",
- "i3",
- "i4",
-};
-
static const char * const guadi2_mme_wap_error_cause[GAUDI2_NUM_OF_MME_WAP_ERR_CAUSE] = {
"WBC ERR RESP_0",
"WBC ERR RESP_1",
@@ -993,6 +984,111 @@ gaudi2_pcie_addr_dec_error_cause[GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE] = {
"TLP is blocked by RR"
};
+static const int gaudi2_queue_id_to_engine_id[] = {
+ [GAUDI2_QUEUE_ID_PDMA_0_0...GAUDI2_QUEUE_ID_PDMA_0_3] = GAUDI2_ENGINE_ID_PDMA_0,
+ [GAUDI2_QUEUE_ID_PDMA_1_0...GAUDI2_QUEUE_ID_PDMA_1_3] = GAUDI2_ENGINE_ID_PDMA_1,
+ [GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_0_3] =
+ GAUDI2_DCORE0_ENGINE_ID_EDMA_0,
+ [GAUDI2_QUEUE_ID_DCORE0_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3] =
+ GAUDI2_DCORE0_ENGINE_ID_EDMA_1,
+ [GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_0_3] =
+ GAUDI2_DCORE1_ENGINE_ID_EDMA_0,
+ [GAUDI2_QUEUE_ID_DCORE1_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE1_EDMA_1_3] =
+ GAUDI2_DCORE1_ENGINE_ID_EDMA_1,
+ [GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_0_3] =
+ GAUDI2_DCORE2_ENGINE_ID_EDMA_0,
+ [GAUDI2_QUEUE_ID_DCORE2_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE2_EDMA_1_3] =
+ GAUDI2_DCORE2_ENGINE_ID_EDMA_1,
+ [GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_0_3] =
+ GAUDI2_DCORE3_ENGINE_ID_EDMA_0,
+ [GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0...GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3] =
+ GAUDI2_DCORE3_ENGINE_ID_EDMA_1,
+ [GAUDI2_QUEUE_ID_DCORE0_MME_0_0...GAUDI2_QUEUE_ID_DCORE0_MME_0_3] =
+ GAUDI2_DCORE0_ENGINE_ID_MME,
+ [GAUDI2_QUEUE_ID_DCORE1_MME_0_0...GAUDI2_QUEUE_ID_DCORE1_MME_0_3] =
+ GAUDI2_DCORE1_ENGINE_ID_MME,
+ [GAUDI2_QUEUE_ID_DCORE2_MME_0_0...GAUDI2_QUEUE_ID_DCORE2_MME_0_3] =
+ GAUDI2_DCORE2_ENGINE_ID_MME,
+ [GAUDI2_QUEUE_ID_DCORE3_MME_0_0...GAUDI2_QUEUE_ID_DCORE3_MME_0_3] =
+ GAUDI2_DCORE3_ENGINE_ID_MME,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_0_0...GAUDI2_QUEUE_ID_DCORE0_TPC_0_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_0,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_1_0...GAUDI2_QUEUE_ID_DCORE0_TPC_1_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_1,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_2_0...GAUDI2_QUEUE_ID_DCORE0_TPC_2_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_2,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_3_0...GAUDI2_QUEUE_ID_DCORE0_TPC_3_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_3,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_4_0...GAUDI2_QUEUE_ID_DCORE0_TPC_4_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_4,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_5_0...GAUDI2_QUEUE_ID_DCORE0_TPC_5_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_5,
+ [GAUDI2_QUEUE_ID_DCORE0_TPC_6_0...GAUDI2_QUEUE_ID_DCORE0_TPC_6_3] =
+ GAUDI2_DCORE0_ENGINE_ID_TPC_6,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_0_0...GAUDI2_QUEUE_ID_DCORE1_TPC_0_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_0,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_1_0...GAUDI2_QUEUE_ID_DCORE1_TPC_1_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_1,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_2_0...GAUDI2_QUEUE_ID_DCORE1_TPC_2_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_2,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_3_0...GAUDI2_QUEUE_ID_DCORE1_TPC_3_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_3,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_4_0...GAUDI2_QUEUE_ID_DCORE1_TPC_4_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_4,
+ [GAUDI2_QUEUE_ID_DCORE1_TPC_5_0...GAUDI2_QUEUE_ID_DCORE1_TPC_5_3] =
+ GAUDI2_DCORE1_ENGINE_ID_TPC_5,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_0_0...GAUDI2_QUEUE_ID_DCORE2_TPC_0_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_0,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_1_0...GAUDI2_QUEUE_ID_DCORE2_TPC_1_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_1,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_2_0...GAUDI2_QUEUE_ID_DCORE2_TPC_2_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_2,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_3_0...GAUDI2_QUEUE_ID_DCORE2_TPC_3_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_3,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_4_0...GAUDI2_QUEUE_ID_DCORE2_TPC_4_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_4,
+ [GAUDI2_QUEUE_ID_DCORE2_TPC_5_0...GAUDI2_QUEUE_ID_DCORE2_TPC_5_3] =
+ GAUDI2_DCORE2_ENGINE_ID_TPC_5,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_0_0...GAUDI2_QUEUE_ID_DCORE3_TPC_0_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_0,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_1_0...GAUDI2_QUEUE_ID_DCORE3_TPC_1_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_1,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_2_0...GAUDI2_QUEUE_ID_DCORE3_TPC_2_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_2,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_3_0...GAUDI2_QUEUE_ID_DCORE3_TPC_3_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_3,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_4_0...GAUDI2_QUEUE_ID_DCORE3_TPC_4_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_4,
+ [GAUDI2_QUEUE_ID_DCORE3_TPC_5_0...GAUDI2_QUEUE_ID_DCORE3_TPC_5_3] =
+ GAUDI2_DCORE3_ENGINE_ID_TPC_5,
+ [GAUDI2_QUEUE_ID_NIC_0_0...GAUDI2_QUEUE_ID_NIC_0_3] = GAUDI2_ENGINE_ID_NIC0_0,
+ [GAUDI2_QUEUE_ID_NIC_1_0...GAUDI2_QUEUE_ID_NIC_1_3] = GAUDI2_ENGINE_ID_NIC0_1,
+ [GAUDI2_QUEUE_ID_NIC_2_0...GAUDI2_QUEUE_ID_NIC_2_3] = GAUDI2_ENGINE_ID_NIC1_0,
+ [GAUDI2_QUEUE_ID_NIC_3_0...GAUDI2_QUEUE_ID_NIC_3_3] = GAUDI2_ENGINE_ID_NIC1_1,
+ [GAUDI2_QUEUE_ID_NIC_4_0...GAUDI2_QUEUE_ID_NIC_4_3] = GAUDI2_ENGINE_ID_NIC2_0,
+ [GAUDI2_QUEUE_ID_NIC_5_0...GAUDI2_QUEUE_ID_NIC_5_3] = GAUDI2_ENGINE_ID_NIC2_1,
+ [GAUDI2_QUEUE_ID_NIC_6_0...GAUDI2_QUEUE_ID_NIC_6_3] = GAUDI2_ENGINE_ID_NIC3_0,
+ [GAUDI2_QUEUE_ID_NIC_7_0...GAUDI2_QUEUE_ID_NIC_7_3] = GAUDI2_ENGINE_ID_NIC3_1,
+ [GAUDI2_QUEUE_ID_NIC_8_0...GAUDI2_QUEUE_ID_NIC_8_3] = GAUDI2_ENGINE_ID_NIC4_0,
+ [GAUDI2_QUEUE_ID_NIC_9_0...GAUDI2_QUEUE_ID_NIC_9_3] = GAUDI2_ENGINE_ID_NIC4_1,
+ [GAUDI2_QUEUE_ID_NIC_10_0...GAUDI2_QUEUE_ID_NIC_10_3] = GAUDI2_ENGINE_ID_NIC5_0,
+ [GAUDI2_QUEUE_ID_NIC_11_0...GAUDI2_QUEUE_ID_NIC_11_3] = GAUDI2_ENGINE_ID_NIC5_1,
+ [GAUDI2_QUEUE_ID_NIC_12_0...GAUDI2_QUEUE_ID_NIC_12_3] = GAUDI2_ENGINE_ID_NIC6_0,
+ [GAUDI2_QUEUE_ID_NIC_13_0...GAUDI2_QUEUE_ID_NIC_13_3] = GAUDI2_ENGINE_ID_NIC6_1,
+ [GAUDI2_QUEUE_ID_NIC_14_0...GAUDI2_QUEUE_ID_NIC_14_3] = GAUDI2_ENGINE_ID_NIC7_0,
+ [GAUDI2_QUEUE_ID_NIC_15_0...GAUDI2_QUEUE_ID_NIC_15_3] = GAUDI2_ENGINE_ID_NIC7_1,
+ [GAUDI2_QUEUE_ID_NIC_16_0...GAUDI2_QUEUE_ID_NIC_16_3] = GAUDI2_ENGINE_ID_NIC8_0,
+ [GAUDI2_QUEUE_ID_NIC_17_0...GAUDI2_QUEUE_ID_NIC_17_3] = GAUDI2_ENGINE_ID_NIC8_1,
+ [GAUDI2_QUEUE_ID_NIC_18_0...GAUDI2_QUEUE_ID_NIC_18_3] = GAUDI2_ENGINE_ID_NIC9_0,
+ [GAUDI2_QUEUE_ID_NIC_19_0...GAUDI2_QUEUE_ID_NIC_19_3] = GAUDI2_ENGINE_ID_NIC9_1,
+ [GAUDI2_QUEUE_ID_NIC_20_0...GAUDI2_QUEUE_ID_NIC_20_3] = GAUDI2_ENGINE_ID_NIC10_0,
+ [GAUDI2_QUEUE_ID_NIC_21_0...GAUDI2_QUEUE_ID_NIC_21_3] = GAUDI2_ENGINE_ID_NIC10_1,
+ [GAUDI2_QUEUE_ID_NIC_22_0...GAUDI2_QUEUE_ID_NIC_22_3] = GAUDI2_ENGINE_ID_NIC11_0,
+ [GAUDI2_QUEUE_ID_NIC_23_0...GAUDI2_QUEUE_ID_NIC_23_3] = GAUDI2_ENGINE_ID_NIC11_1,
+ [GAUDI2_QUEUE_ID_ROT_0_0...GAUDI2_QUEUE_ID_ROT_0_3] = GAUDI2_ENGINE_ID_ROT_0,
+ [GAUDI2_QUEUE_ID_ROT_1_0...GAUDI2_QUEUE_ID_ROT_1_3] = GAUDI2_ENGINE_ID_ROT_1,
+};
+
const u32 gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_SIZE] = {
[GAUDI2_QUEUE_ID_PDMA_0_0] = mmPDMA0_QM_BASE,
[GAUDI2_QUEUE_ID_PDMA_0_1] = mmPDMA0_QM_BASE,
@@ -2001,7 +2097,8 @@ enum razwi_event_sources {
RAZWI_PDMA,
RAZWI_NIC,
RAZWI_DEC,
- RAZWI_ROT
+ RAZWI_ROT,
+ RAZWI_ARC_FARM
};
struct hbm_mc_error_causes {
@@ -2431,7 +2528,7 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev)
prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE;
prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
- strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
+ strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
prop->mme_master_slave_mode = 1;
@@ -2884,7 +2981,8 @@ static int gaudi2_cpucp_info_get(struct hl_device *hdev)
}
if (!strlen(prop->cpucp_info.card_name))
- strncpy(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);
+ strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME,
+ CARD_NAME_MAX_LEN);
/* Overwrite binning masks with the actual binning values from F/W */
hdev->dram_binning = prop->cpucp_info.dram_binning_mask;
@@ -4077,6 +4175,8 @@ static const char *gaudi2_irq_name(u16 irq_number)
return "gaudi2 unexpected error";
case GAUDI2_IRQ_NUM_USER_FIRST ... GAUDI2_IRQ_NUM_USER_LAST:
return "gaudi2 user completion";
+ case GAUDI2_IRQ_NUM_EQ_ERROR:
+ return "gaudi2 eq error";
default:
return "invalid";
}
@@ -4127,9 +4227,7 @@ static int gaudi2_dec_enable_msix(struct hl_device *hdev)
rc = request_irq(irq, hl_irq_handler_dec_abnrm, 0,
gaudi2_irq_name(i), (void *) dec);
} else {
- rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
- hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
- gaudi2_irq_name(i),
+ rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i),
(void *) &hdev->user_interrupt[dec->core_id]);
}
@@ -4187,17 +4285,17 @@ static int gaudi2_enable_msix(struct hl_device *hdev)
}
irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);
- rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
- hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
- gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT), &hdev->tpc_interrupt);
+ rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
+ gaudi2_irq_name(GAUDI2_IRQ_NUM_TPC_ASSERT),
+ &hdev->tpc_interrupt);
if (rc) {
dev_err(hdev->dev, "Failed to request IRQ %d", irq);
goto free_dec_irq;
}
irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);
- rc = request_irq(irq, hl_irq_handler_user_interrupt, 0,
- gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR),
+ rc = request_threaded_irq(irq, NULL, hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
+ gaudi2_irq_name(GAUDI2_IRQ_NUM_UNEXPECTED_ERROR),
&hdev->unexpected_error_interrupt);
if (rc) {
dev_err(hdev->dev, "Failed to request IRQ %d", irq);
@@ -4209,16 +4307,23 @@ static int gaudi2_enable_msix(struct hl_device *hdev)
i++, j++, user_irq_init_cnt++) {
irq = pci_irq_vector(hdev->pdev, i);
- rc = request_threaded_irq(irq, hl_irq_handler_user_interrupt,
- hl_irq_user_interrupt_thread_handler, IRQF_ONESHOT,
- gaudi2_irq_name(i), &hdev->user_interrupt[j]);
-
+ rc = request_irq(irq, hl_irq_user_interrupt_handler, 0, gaudi2_irq_name(i),
+ &hdev->user_interrupt[j]);
if (rc) {
dev_err(hdev->dev, "Failed to request IRQ %d", irq);
goto free_user_irq;
}
}
+ irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR);
+ rc = request_threaded_irq(irq, NULL, hl_irq_eq_error_interrupt_thread_handler,
+ IRQF_ONESHOT, gaudi2_irq_name(GAUDI2_IRQ_NUM_EQ_ERROR),
+ hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+ goto free_user_irq;
+ }
+
gaudi2->hw_cap_initialized |= HW_CAP_MSIX;
return 0;
@@ -4278,6 +4383,7 @@ static void gaudi2_sync_irqs(struct hl_device *hdev)
}
synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));
+ synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR));
}
static void gaudi2_disable_msix(struct hl_device *hdev)
@@ -4314,6 +4420,9 @@ static void gaudi2_disable_msix(struct hl_device *hdev)
cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];
free_irq(irq, cq);
+ irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR);
+ free_irq(irq, hdev);
+
pci_free_irq_vectors(hdev->pdev);
gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX;
@@ -4716,6 +4825,8 @@ static void gaudi2_init_firmware_preload_params(struct hl_device *hdev)
pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;
pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;
pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC;
+ pre_fw_load->wait_for_preboot_extended_timeout =
+ GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC;
}
static void gaudi2_init_firmware_loader(struct hl_device *hdev)
@@ -6157,17 +6268,14 @@ static int gaudi2_get_soft_rst_done_indication(struct hl_device *hdev, u32 poll_
static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset,
u32 poll_timeout_us)
{
- struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
- int rc = 0;
+ int rc;
if (!driver_performs_reset) {
if (hl_is_fw_sw_ver_below(hdev, 1, 10)) {
/* set SP to indicate reset request sent to FW */
- if (dyn_regs->cpu_rst_status)
- WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA);
- else
- WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA);
- WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq),
+ WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA);
+
+ WREG32(mmGIC_HOST_SOFT_RST_IRQ_POLL_REG,
gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id);
/* wait for f/w response */
@@ -6623,24 +6731,6 @@ static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t s
hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
}
-static dma_addr_t gaudi2_dma_map_single(struct hl_device *hdev, void *addr, int len,
- enum dma_data_direction dir)
-{
- dma_addr_t dma_addr;
-
- dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, dir);
- if (unlikely(dma_mapping_error(&hdev->pdev->dev, dma_addr)))
- return 0;
-
- return dma_addr;
-}
-
-static void gaudi2_dma_unmap_single(struct hl_device *hdev, dma_addr_t addr, int len,
- enum dma_data_direction dir)
-{
- dma_unmap_single(&hdev->pdev->dev, addr, len, dir);
-}
-
static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser)
{
struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
@@ -7703,11 +7793,13 @@ static inline bool is_info_event(u32 event)
switch (event) {
case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
+ case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY ... GAUDI2_EVENT_ARC_PWR_RD_MODE3:
/* return in case of NIC status event - these events are received periodically and not as
* an indication to an error.
*/
case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1:
+ case GAUDI2_EVENT_ARC_EQ_HEARTBEAT:
return true;
default:
return false;
@@ -7739,21 +7831,34 @@ static void gaudi2_print_event(struct hl_device *hdev, u16 event_type,
static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
struct hl_eq_ecc_data *ecc_data)
{
- u64 ecc_address = 0, ecc_syndrom = 0;
+ u64 ecc_address = 0, ecc_syndrome = 0;
u8 memory_wrapper_idx = 0;
+ bool has_block_id = false;
+ u16 block_id;
+
+ if (!hl_is_fw_sw_ver_below(hdev, 1, 12))
+ has_block_id = true;
ecc_address = le64_to_cpu(ecc_data->ecc_address);
- ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);
+ ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom);
memory_wrapper_idx = ecc_data->memory_wrapper_idx;
- gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
- "ECC error detected. address: %#llx. Syndrom: %#llx. block id %u. critical %u.",
- ecc_address, ecc_syndrom, memory_wrapper_idx, ecc_data->is_critical);
+ if (has_block_id) {
+ block_id = le16_to_cpu(ecc_data->block_id);
+ gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
+ "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. block id %#x. critical %u.",
+ ecc_address, ecc_syndrome, memory_wrapper_idx, block_id,
+ ecc_data->is_critical);
+ } else {
+ gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,
+ "ECC error detected. address: %#llx. Syndrome: %#llx. wrapper id %u. critical %u.",
+ ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical);
+ }
return !!ecc_data->is_critical;
}
-static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base)
+static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u64 event_mask)
{
u32 lo, hi, cq_ptr_size, arc_cq_ptr_size;
u64 cq_ptr, arc_cq_ptr, cp_current_inst;
@@ -7775,10 +7880,22 @@ static void print_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base)
dev_info(hdev->dev,
"LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n",
cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst);
+
+ if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) {
+ if (arc_cq_ptr) {
+ hdev->captured_err_info.undef_opcode.cq_addr = arc_cq_ptr;
+ hdev->captured_err_info.undef_opcode.cq_size = arc_cq_ptr_size;
+ } else {
+ hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr;
+ hdev->captured_err_info.undef_opcode.cq_size = cq_ptr_size;
+ }
+
+ hdev->captured_err_info.undef_opcode.stream_id = QMAN_STREAMS;
+ }
}
static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type,
- u64 qman_base, u32 qid_base)
+ u64 qman_base, u32 qid_base, u64 *event_mask)
{
u32 i, j, glbl_sts_val, arb_err_val, num_error_causes, error_count = 0;
u64 glbl_sts_addr, arb_err_addr;
@@ -7812,8 +7929,22 @@ static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type
error_count++;
}
- if (i == QMAN_STREAMS)
- print_lower_qman_data_on_err(hdev, qman_base);
+ if (i == QMAN_STREAMS && error_count) {
+ /* check for undefined opcode */
+ if (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK &&
+ hdev->captured_err_info.undef_opcode.write_enable) {
+ memset(&hdev->captured_err_info.undef_opcode, 0,
+ sizeof(hdev->captured_err_info.undef_opcode));
+
+ hdev->captured_err_info.undef_opcode.write_enable = false;
+ hdev->captured_err_info.undef_opcode.timestamp = ktime_get();
+ hdev->captured_err_info.undef_opcode.engine_id =
+ gaudi2_queue_id_to_engine_id[qid_base];
+ *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE;
+ }
+
+ handle_lower_qman_data_on_err(hdev, qman_base, *event_mask);
+ }
}
arb_err_val = RREG32(arb_err_addr);
@@ -7927,6 +8058,9 @@ static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev,
case RAZWI_ROT:
return GAUDI2_ENGINE_ID_ROT_0 + module_idx;
+ case RAZWI_ARC_FARM:
+ return GAUDI2_ENGINE_ID_ARC_FARM;
+
default:
return GAUDI2_ENGINE_ID_SIZE;
}
@@ -8036,6 +8170,11 @@ static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
lbw_rtr_id = gaudi2_rot_initiator_lbw_rtr_id[module_idx];
sprintf(initiator_name, "ROT_%u", module_idx);
break;
+ case RAZWI_ARC_FARM:
+ lbw_rtr_id = DCORE1_RTR5;
+ hbw_rtr_id = DCORE1_RTR7;
+ sprintf(initiator_name, "ARC_FARM_%u", module_idx);
+ break;
default:
return;
}
@@ -8149,11 +8288,11 @@ static int gaudi2_psoc_razwi_get_engines(struct gaudi2_razwi_info *razwi_info, u
eng_id[num_of_eng] = razwi_info[i].eng_id;
base[num_of_eng] = razwi_info[i].rtr_ctrl;
if (!num_of_eng)
- str_size += snprintf(eng_name + str_size,
+ str_size += scnprintf(eng_name + str_size,
PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s",
razwi_info[i].eng_name);
else
- str_size += snprintf(eng_name + str_size,
+ str_size += scnprintf(eng_name + str_size,
PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s",
razwi_info[i].eng_name);
num_of_eng++;
@@ -8475,7 +8614,8 @@ static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e
return 0;
}
- error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base, qid_base);
+ error_count = gaudi2_handle_qman_err_generic(hdev, event_type, qman_base,
+ qid_base, event_mask);
/* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */
if (event_type >= GAUDI2_EVENT_HDMA2_QM && event_type <= GAUDI2_EVENT_HDMA5_QM) {
@@ -8488,7 +8628,7 @@ static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e
return error_count;
}
-static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type)
+static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
{
u32 i, sts_val, sts_clr_val, error_count = 0, arc_farm;
@@ -8510,6 +8650,7 @@ static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type
sts_clr_val);
}
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ARC_FARM, 0, 0, event_mask);
hl_check_for_glbl_errors(hdev);
return error_count;
@@ -8649,21 +8790,16 @@ static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event
return error_count;
}
-static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type,
- u64 intr_cause_data)
+static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type)
{
- int i, error_count = 0;
-
- for (i = 0 ; i < GAUDI2_NUM_OF_MME_SBTE_ERR_CAUSE ; i++)
- if (intr_cause_data & BIT(i)) {
- gaudi2_print_event(hdev, event_type, true,
- "err cause: %s", guadi2_mme_sbte_error_cause[i]);
- error_count++;
- }
-
+ /*
+ * We have a single error cause here but the report mechanism is
+ * buggy. Hence there is no good reason to fetch the cause so we
+ * just check for glbl_errors and exit.
+ */
hl_check_for_glbl_errors(hdev);
- return error_count;
+ return GAUDI2_NA_EVENT_CAUSE;
}
static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type,
@@ -9460,6 +9596,176 @@ static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type,
}
}
+static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type)
+{
+ enum gaudi2_block_types type = GAUDI2_BLOCK_TYPE_MAX;
+ u16 index;
+
+ switch (event_type) {
+ case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
+ index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
+ type = GAUDI2_BLOCK_TYPE_TPC;
+ break;
+ case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_TPC24_QM:
+ index = event_type - GAUDI2_EVENT_TPC0_QM;
+ type = GAUDI2_BLOCK_TYPE_TPC;
+ break;
+ case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_MME0_SPI_BASE ... GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID:
+ case GAUDI2_EVENT_MME0_QM:
+ index = 0;
+ type = GAUDI2_BLOCK_TYPE_MME;
+ break;
+ case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_MME1_SPI_BASE ... GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID:
+ case GAUDI2_EVENT_MME1_QM:
+ index = 1;
+ type = GAUDI2_BLOCK_TYPE_MME;
+ break;
+ case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_CTRL_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_MME2_SPI_BASE ... GAUDI2_EVENT_MME2_WAP_SOURCE_RESULT_INVALID:
+ case GAUDI2_EVENT_MME2_QM:
+ index = 2;
+ type = GAUDI2_BLOCK_TYPE_MME;
+ break;
+ case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_CTRL_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_MME3_SPI_BASE ... GAUDI2_EVENT_MME3_WAP_SOURCE_RESULT_INVALID:
+ case GAUDI2_EVENT_MME3_QM:
+ index = 3;
+ type = GAUDI2_BLOCK_TYPE_MME;
+ break;
+ case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP:
+ case GAUDI2_EVENT_KDMA_BM_SPMU:
+ case GAUDI2_EVENT_KDMA0_CORE:
+ return GAUDI2_ENGINE_ID_KDMA;
+ case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
+ case GAUDI2_EVENT_PDMA0_CORE:
+ case GAUDI2_EVENT_PDMA0_BM_SPMU:
+ case GAUDI2_EVENT_PDMA0_QM:
+ return GAUDI2_ENGINE_ID_PDMA_0;
+ case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
+ case GAUDI2_EVENT_PDMA1_CORE:
+ case GAUDI2_EVENT_PDMA1_BM_SPMU:
+ case GAUDI2_EVENT_PDMA1_QM:
+ return GAUDI2_ENGINE_ID_PDMA_1;
+ case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE:
+ index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;
+ type = GAUDI2_BLOCK_TYPE_DEC;
+ break;
+ case GAUDI2_EVENT_DEC0_SPI ... GAUDI2_EVENT_DEC9_BMON_SPMU:
+ index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1;
+ type = GAUDI2_BLOCK_TYPE_DEC;
+ break;
+ case GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_NIC11_AXI_ERROR_RESPONSE:
+ index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE;
+ return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2);
+ case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1:
+ index = event_type - GAUDI2_EVENT_NIC0_QM0;
+ return GAUDI2_ENGINE_ID_NIC0_0 + index;
+ case GAUDI2_EVENT_NIC0_BMON_SPMU ... GAUDI2_EVENT_NIC11_SW_ERROR:
+ index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU;
+ return GAUDI2_ENGINE_ID_NIC0_0 + (index * 2);
+ case GAUDI2_EVENT_TPC0_BMON_SPMU ... GAUDI2_EVENT_TPC24_KERNEL_ERR:
+ index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1;
+ type = GAUDI2_BLOCK_TYPE_TPC;
+ break;
+ case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_ROTATOR0_BMON_SPMU:
+ case GAUDI2_EVENT_ROTATOR0_ROT0_QM:
+ return GAUDI2_ENGINE_ID_ROT_0;
+ case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
+ case GAUDI2_EVENT_ROTATOR1_BMON_SPMU:
+ case GAUDI2_EVENT_ROTATOR1_ROT1_QM:
+ return GAUDI2_ENGINE_ID_ROT_1;
+ case GAUDI2_EVENT_HDMA0_BM_SPMU:
+ case GAUDI2_EVENT_HDMA0_QM:
+ case GAUDI2_EVENT_HDMA0_CORE:
+ return GAUDI2_DCORE0_ENGINE_ID_EDMA_0;
+ case GAUDI2_EVENT_HDMA1_BM_SPMU:
+ case GAUDI2_EVENT_HDMA1_QM:
+ case GAUDI2_EVENT_HDMA1_CORE:
+ return GAUDI2_DCORE0_ENGINE_ID_EDMA_1;
+ case GAUDI2_EVENT_HDMA2_BM_SPMU:
+ case GAUDI2_EVENT_HDMA2_QM:
+ case GAUDI2_EVENT_HDMA2_CORE:
+ return GAUDI2_DCORE1_ENGINE_ID_EDMA_0;
+ case GAUDI2_EVENT_HDMA3_BM_SPMU:
+ case GAUDI2_EVENT_HDMA3_QM:
+ case GAUDI2_EVENT_HDMA3_CORE:
+ return GAUDI2_DCORE1_ENGINE_ID_EDMA_1;
+ case GAUDI2_EVENT_HDMA4_BM_SPMU:
+ case GAUDI2_EVENT_HDMA4_QM:
+ case GAUDI2_EVENT_HDMA4_CORE:
+ return GAUDI2_DCORE2_ENGINE_ID_EDMA_0;
+ case GAUDI2_EVENT_HDMA5_BM_SPMU:
+ case GAUDI2_EVENT_HDMA5_QM:
+ case GAUDI2_EVENT_HDMA5_CORE:
+ return GAUDI2_DCORE2_ENGINE_ID_EDMA_1;
+ case GAUDI2_EVENT_HDMA6_BM_SPMU:
+ case GAUDI2_EVENT_HDMA6_QM:
+ case GAUDI2_EVENT_HDMA6_CORE:
+ return GAUDI2_DCORE3_ENGINE_ID_EDMA_0;
+ case GAUDI2_EVENT_HDMA7_BM_SPMU:
+ case GAUDI2_EVENT_HDMA7_QM:
+ case GAUDI2_EVENT_HDMA7_CORE:
+ return GAUDI2_DCORE3_ENGINE_ID_EDMA_1;
+ default:
+ break;
+ }
+
+ switch (type) {
+ case GAUDI2_BLOCK_TYPE_TPC:
+ switch (index) {
+ case TPC_ID_DCORE0_TPC0 ... TPC_ID_DCORE0_TPC5:
+ return GAUDI2_DCORE0_ENGINE_ID_TPC_0 + index;
+ case TPC_ID_DCORE1_TPC0 ... TPC_ID_DCORE1_TPC5:
+ return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0;
+ case TPC_ID_DCORE2_TPC0 ... TPC_ID_DCORE2_TPC5:
+ return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0;
+ case TPC_ID_DCORE3_TPC0 ... TPC_ID_DCORE3_TPC5:
+ return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0;
+ default:
+ break;
+ }
+ break;
+ case GAUDI2_BLOCK_TYPE_MME:
+ switch (index) {
+ case MME_ID_DCORE0: return GAUDI2_DCORE0_ENGINE_ID_MME;
+ case MME_ID_DCORE1: return GAUDI2_DCORE1_ENGINE_ID_MME;
+ case MME_ID_DCORE2: return GAUDI2_DCORE2_ENGINE_ID_MME;
+ case MME_ID_DCORE3: return GAUDI2_DCORE3_ENGINE_ID_MME;
+ default:
+ break;
+ }
+ break;
+ case GAUDI2_BLOCK_TYPE_DEC:
+ switch (index) {
+ case DEC_ID_DCORE0_DEC0: return GAUDI2_DCORE0_ENGINE_ID_DEC_0;
+ case DEC_ID_DCORE0_DEC1: return GAUDI2_DCORE0_ENGINE_ID_DEC_1;
+ case DEC_ID_DCORE1_DEC0: return GAUDI2_DCORE1_ENGINE_ID_DEC_0;
+ case DEC_ID_DCORE1_DEC1: return GAUDI2_DCORE1_ENGINE_ID_DEC_1;
+ case DEC_ID_DCORE2_DEC0: return GAUDI2_DCORE2_ENGINE_ID_DEC_0;
+ case DEC_ID_DCORE2_DEC1: return GAUDI2_DCORE2_ENGINE_ID_DEC_1;
+ case DEC_ID_DCORE3_DEC0: return GAUDI2_DCORE3_ENGINE_ID_DEC_0;
+ case DEC_ID_DCORE3_DEC1: return GAUDI2_DCORE3_ENGINE_ID_DEC_1;
+ case DEC_ID_PCIE_VDEC0: return GAUDI2_PCIE_ENGINE_ID_DEC_0;
+ case DEC_ID_PCIE_VDEC1: return GAUDI2_PCIE_ENGINE_ID_DEC_1;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return U16_MAX;
+}
+
+static void hl_eq_heartbeat_event_handle(struct hl_device *hdev)
+{
+ hdev->eq_heartbeat_received = true;
+}
+
static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
{
struct gaudi2_device *gaudi2 = hdev->asic_specific;
@@ -9501,7 +9807,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
break;
case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0:
- error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type);
+ error_count = gaudi2_handle_arc_farm_sei_err(hdev, event_type, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -9724,8 +10030,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP:
case GAUDI2_EVENT_MME2_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME2_SBTE4_AXI_ERR_RSP:
case GAUDI2_EVENT_MME3_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME3_SBTE4_AXI_ERR_RSP:
- error_count = gaudi2_handle_mme_sbte_err(hdev, event_type,
- le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
+ error_count = gaudi2_handle_mme_sbte_err(hdev, event_type);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B:
@@ -9875,6 +10180,21 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
is_critical = true;
break;
+ case GAUDI2_EVENT_ARC_PWR_BRK_ENTRY:
+ case GAUDI2_EVENT_ARC_PWR_BRK_EXT:
+ case GAUDI2_EVENT_ARC_PWR_RD_MODE0:
+ case GAUDI2_EVENT_ARC_PWR_RD_MODE1:
+ case GAUDI2_EVENT_ARC_PWR_RD_MODE2:
+ case GAUDI2_EVENT_ARC_PWR_RD_MODE3:
+ error_count = GAUDI2_NA_EVENT_CAUSE;
+ dev_info_ratelimited(hdev->dev, "%s event received\n",
+ gaudi2_irq_map_table[event_type].name);
+ break;
+
+ case GAUDI2_EVENT_ARC_EQ_HEARTBEAT:
+ hl_eq_heartbeat_event_handle(hdev);
+ error_count = GAUDI2_NA_EVENT_CAUSE;
+ break;
default:
if (gaudi2_irq_map_table[event_type].valid) {
dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
@@ -9883,6 +10203,9 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
}
}
+ if (event_mask & HL_NOTIFIER_EVENT_USER_ENGINE_ERR)
+ hl_capture_engine_err(hdev, event_id_to_engine_id(hdev, event_type), error_count);
+
/* Make sure to dump an error in case no error cause was printed so far.
* Note that although we have counted the errors, we use this number as
* a boolean.
@@ -10523,6 +10846,9 @@ static int gaudi2_ctx_init(struct hl_ctx *ctx)
{
int rc;
+ if (ctx->asid == HL_KERNEL_ASID_ID)
+ return 0;
+
rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);
if (rc)
return rc;
@@ -11014,6 +11340,7 @@ static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64
static void gaudi2_get_msi_info(__le32 *table)
{
table[CPUCP_EVENT_QUEUE_MSI_TYPE] = cpu_to_le32(GAUDI2_EVENT_QUEUE_MSIX_IDX);
+ table[CPUCP_EVENT_QUEUE_ERR_MSI_TYPE] = cpu_to_le32(GAUDI2_IRQ_NUM_EQ_ERROR);
}
static int gaudi2_map_pll_idx_to_fw_idx(u32 pll_idx)
@@ -11170,11 +11497,9 @@ static const struct hl_asic_funcs gaudi2_funcs = {
.asic_dma_pool_free = gaudi2_dma_pool_free,
.cpu_accessible_dma_pool_alloc = gaudi2_cpu_accessible_dma_pool_alloc,
.cpu_accessible_dma_pool_free = gaudi2_cpu_accessible_dma_pool_free,
- .asic_dma_unmap_single = gaudi2_dma_unmap_single,
- .asic_dma_map_single = gaudi2_dma_map_single,
- .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
+ .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable,
.cs_parser = gaudi2_cs_parser,
- .asic_dma_map_sgtable = hl_dma_map_sgtable,
+ .dma_map_sgtable = hl_asic_dma_map_sgtable,
.add_end_of_cb_packets = NULL,
.update_eq_ci = gaudi2_update_eq_ci,
.context_switch = gaudi2_context_switch,
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2P.h b/drivers/accel/habanalabs/gaudi2/gaudi2P.h
index 5f3ce086928e..9b9eef0d97d6 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2P.h
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2P.h
@@ -10,7 +10,7 @@
#include <uapi/drm/habanalabs_accel.h>
#include "../common/habanalabs.h"
-#include "../include/common/hl_boot_if.h"
+#include <linux/habanalabs/hl_boot_if.h>
#include "../include/gaudi2/gaudi2.h"
#include "../include/gaudi2/gaudi2_packets.h"
#include "../include/gaudi2/gaudi2_fw_if.h"
@@ -84,6 +84,7 @@
#define CORESIGHT_TIMEOUT_USEC 100000 /* 100 ms */
#define GAUDI2_PREBOOT_REQ_TIMEOUT_USEC 25000000 /* 25s */
+#define GAUDI2_PREBOOT_EXTENDED_REQ_TIMEOUT_USEC 85000000 /* 85s */
#define GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC 10000000 /* 10s */
@@ -419,6 +420,7 @@ enum gaudi2_irq_num {
GAUDI2_IRQ_NUM_NIC_PORT_FIRST,
GAUDI2_IRQ_NUM_NIC_PORT_LAST = (GAUDI2_IRQ_NUM_NIC_PORT_FIRST + NIC_NUMBER_OF_PORTS - 1),
GAUDI2_IRQ_NUM_TPC_ASSERT,
+ GAUDI2_IRQ_NUM_EQ_ERROR,
GAUDI2_IRQ_NUM_RESERVED_FIRST,
GAUDI2_IRQ_NUM_RESERVED_LAST = (GAUDI2_MSIX_ENTRIES - GAUDI2_TOTAL_USER_INTERRUPTS - 1),
GAUDI2_IRQ_NUM_UNEXPECTED_ERROR = RESERVED_MSIX_UNEXPECTED_USER_ERROR_INTERRUPT,
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
index 25b5368f37dd..2423620ff358 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2_coresight.c
@@ -151,8 +151,8 @@ static u64 debug_stm_regs[GAUDI2_STM_LAST + 1] = {
[GAUDI2_STM_DCORE3_VDEC1_CS] = mmDCORE3_VDEC1_CS_STM_BASE,
[GAUDI2_STM_PCIE] = mmPCIE_STM_BASE,
[GAUDI2_STM_PSOC] = mmPSOC_STM_BASE,
- [GAUDI2_STM_PSOC_ARC0_CS] = mmPSOC_ARC0_CS_STM_BASE,
- [GAUDI2_STM_PSOC_ARC1_CS] = mmPSOC_ARC1_CS_STM_BASE,
+ [GAUDI2_STM_PSOC_ARC0_CS] = 0,
+ [GAUDI2_STM_PSOC_ARC1_CS] = 0,
[GAUDI2_STM_PDMA0_CS] = mmPDMA0_CS_STM_BASE,
[GAUDI2_STM_PDMA1_CS] = mmPDMA1_CS_STM_BASE,
[GAUDI2_STM_CPU] = mmCPU_STM_BASE,
@@ -293,8 +293,8 @@ static u64 debug_etf_regs[GAUDI2_ETF_LAST + 1] = {
[GAUDI2_ETF_DCORE3_VDEC1_CS] = mmDCORE3_VDEC1_CS_ETF_BASE,
[GAUDI2_ETF_PCIE] = mmPCIE_ETF_BASE,
[GAUDI2_ETF_PSOC] = mmPSOC_ETF_BASE,
- [GAUDI2_ETF_PSOC_ARC0_CS] = mmPSOC_ARC0_CS_ETF_BASE,
- [GAUDI2_ETF_PSOC_ARC1_CS] = mmPSOC_ARC1_CS_ETF_BASE,
+ [GAUDI2_ETF_PSOC_ARC0_CS] = 0,
+ [GAUDI2_ETF_PSOC_ARC1_CS] = 0,
[GAUDI2_ETF_PDMA0_CS] = mmPDMA0_CS_ETF_BASE,
[GAUDI2_ETF_PDMA1_CS] = mmPDMA1_CS_ETF_BASE,
[GAUDI2_ETF_CPU_0] = mmCPU_ETF_0_BASE,
@@ -436,8 +436,8 @@ static u64 debug_funnel_regs[GAUDI2_FUNNEL_LAST + 1] = {
[GAUDI2_FUNNEL_DCORE3_RTR6] = mmDCORE3_RTR6_FUNNEL_BASE,
[GAUDI2_FUNNEL_DCORE3_RTR7] = mmDCORE3_RTR7_FUNNEL_BASE,
[GAUDI2_FUNNEL_PSOC] = mmPSOC_FUNNEL_BASE,
- [GAUDI2_FUNNEL_PSOC_ARC0] = mmPSOC_ARC0_FUNNEL_BASE,
- [GAUDI2_FUNNEL_PSOC_ARC1] = mmPSOC_ARC1_FUNNEL_BASE,
+ [GAUDI2_FUNNEL_PSOC_ARC0] = 0,
+ [GAUDI2_FUNNEL_PSOC_ARC1] = 0,
[GAUDI2_FUNNEL_XDMA] = mmXDMA_FUNNEL_BASE,
[GAUDI2_FUNNEL_CPU] = mmCPU_FUNNEL_BASE,
[GAUDI2_FUNNEL_PMMU] = mmPMMU_FUNNEL_BASE,
@@ -766,10 +766,10 @@ static u64 debug_bmon_regs[GAUDI2_BMON_LAST + 1] = {
[GAUDI2_BMON_PCIE_MSTR_RD] = mmPCIE_BMON_MSTR_RD_BASE,
[GAUDI2_BMON_PCIE_SLV_WR] = mmPCIE_BMON_SLV_WR_BASE,
[GAUDI2_BMON_PCIE_SLV_RD] = mmPCIE_BMON_SLV_RD_BASE,
- [GAUDI2_BMON_PSOC_ARC0_0] = mmPSOC_ARC0_BMON_0_BASE,
- [GAUDI2_BMON_PSOC_ARC0_1] = mmPSOC_ARC0_BMON_1_BASE,
- [GAUDI2_BMON_PSOC_ARC1_0] = mmPSOC_ARC1_BMON_0_BASE,
- [GAUDI2_BMON_PSOC_ARC1_1] = mmPSOC_ARC1_BMON_1_BASE,
+ [GAUDI2_BMON_PSOC_ARC0_0] = 0,
+ [GAUDI2_BMON_PSOC_ARC0_1] = 0,
+ [GAUDI2_BMON_PSOC_ARC1_0] = 0,
+ [GAUDI2_BMON_PSOC_ARC1_1] = 0,
[GAUDI2_BMON_PDMA0_0] = mmPDMA0_BMON_0_BASE,
[GAUDI2_BMON_PDMA0_1] = mmPDMA0_BMON_1_BASE,
[GAUDI2_BMON_PDMA1_0] = mmPDMA1_BMON_0_BASE,
@@ -968,8 +968,8 @@ static u64 debug_spmu_regs[GAUDI2_SPMU_LAST + 1] = {
[GAUDI2_SPMU_DCORE3_VDEC0_CS] = mmDCORE3_VDEC0_CS_SPMU_BASE,
[GAUDI2_SPMU_DCORE3_VDEC1_CS] = mmDCORE3_VDEC1_CS_SPMU_BASE,
[GAUDI2_SPMU_PCIE] = mmPCIE_SPMU_BASE,
- [GAUDI2_SPMU_PSOC_ARC0_CS] = mmPSOC_ARC0_CS_SPMU_BASE,
- [GAUDI2_SPMU_PSOC_ARC1_CS] = mmPSOC_ARC1_CS_SPMU_BASE,
+ [GAUDI2_SPMU_PSOC_ARC0_CS] = 0,
+ [GAUDI2_SPMU_PSOC_ARC1_CS] = 0,
[GAUDI2_SPMU_PDMA0_CS] = mmPDMA0_CS_SPMU_BASE,
[GAUDI2_SPMU_PDMA1_CS] = mmPDMA1_CS_SPMU_BASE,
[GAUDI2_SPMU_PMMU_CS] = mmPMMU_CS_SPMU_BASE,
@@ -2092,6 +2092,11 @@ static int gaudi2_config_etf(struct hl_device *hdev, struct hl_debug_params *par
if (rc)
return -EIO;
+ val = RREG32(base_reg + mmETF_CTL_OFFSET);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
val = RREG32(base_reg + mmETF_FFCR_OFFSET);
val |= 0x1000;
WREG32(base_reg + mmETF_FFCR_OFFSET, val);
@@ -2120,10 +2125,17 @@ static int gaudi2_config_etf(struct hl_device *hdev, struct hl_debug_params *par
if (!input)
return -EINVAL;
+ val = RREG32(base_reg + mmETF_RSZ_OFFSET) << 2;
+ if (val) {
+ val = ffs(val);
+ WREG32(base_reg + mmETF_PSCR_OFFSET, val);
+ } else {
+ WREG32(base_reg + mmETF_PSCR_OFFSET, 0x10);
+ }
+
WREG32(base_reg + mmETF_BUFWM_OFFSET, 0x3FFC);
WREG32(base_reg + mmETF_MODE_OFFSET, input->sink_mode);
WREG32(base_reg + mmETF_FFCR_OFFSET, 0x4001);
- WREG32(base_reg + mmETF_PSCR_OFFSET, 0x10);
WREG32(base_reg + mmETF_CTL_OFFSET, 1);
} else {
WREG32(base_reg + mmETF_BUFWM_OFFSET, 0);
@@ -2189,6 +2201,11 @@ static int gaudi2_config_etr(struct hl_device *hdev, struct hl_ctx *ctx,
if (rc)
return -EIO;
+ val = RREG32(mmPSOC_ETR_CTL);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
val = RREG32(mmPSOC_ETR_FFCR);
val |= 0x1000;
WREG32(mmPSOC_ETR_FFCR, val);
@@ -2483,7 +2500,8 @@ static int gaudi2_config_spmu(struct hl_device *hdev, struct hl_debug_params *pa
* set enabled events mask based on input->event_types_num
*/
event_mask = 0x80000000;
- event_mask |= GENMASK(input->event_types_num, 0);
+ if (input->event_types_num)
+ event_mask |= GENMASK(input->event_types_num - 1, 0);
WREG32(base_reg + mmSPMU_PMCNTENSET_EL0_OFFSET, event_mask);
} else {
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2_security.c b/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
index 2742b1f801eb..34bf80c5a44b 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2_security.c
@@ -1601,6 +1601,7 @@ static const u32 gaudi2_pb_dcr0_tpc0_unsecured_regs[] = {
mmDCORE0_TPC0_CFG_KERNEL_SRF_30,
mmDCORE0_TPC0_CFG_KERNEL_SRF_31,
mmDCORE0_TPC0_CFG_TPC_SB_L0CD,
+ mmDCORE0_TPC0_CFG_TPC_COUNT,
mmDCORE0_TPC0_CFG_TPC_ID,
mmDCORE0_TPC0_CFG_QM_KERNEL_ID_INC,
mmDCORE0_TPC0_CFG_QM_TID_BASE_SIZE_HIGH_DIM_0,
@@ -2907,7 +2908,7 @@ static void gaudi2_init_lbw_range_registers_secure(struct hl_device *hdev)
* - range 11: NIC11_CFG + *_DBG (not including TPC_DBG)
*
* If F/W security is not enabled:
- * - ranges 12,13: PSOC_CFG (excluding PSOC_TIMESTAMP)
+ * - ranges 12,13: PSOC_CFG (excluding PSOC_TIMESTAMP, PSOC_EFUSE and PSOC_GLOBAL_CONF)
*/
u64 lbw_range_min_short[] = {
mmNIC0_TX_AXUSER_BASE,
@@ -2923,7 +2924,7 @@ static void gaudi2_init_lbw_range_registers_secure(struct hl_device *hdev)
mmNIC10_TX_AXUSER_BASE,
mmNIC11_TX_AXUSER_BASE,
mmPSOC_I2C_M0_BASE,
- mmPSOC_EFUSE_BASE
+ mmPSOC_GPIO0_BASE
};
u64 lbw_range_max_short[] = {
mmNIC0_MAC_CH3_MAC_PCS_BASE + HL_BLOCK_SIZE,
@@ -3219,6 +3220,7 @@ static void gaudi2_init_range_registers(struct hl_device *hdev)
*/
static int gaudi2_init_protection_bits(struct hl_device *hdev)
{
+ u32 *user_regs_array = NULL, user_regs_array_size = 0, engine_core_intr_reg;
struct asic_fixed_properties *prop = &hdev->asic_prop;
u32 instance_offset;
int rc = 0;
@@ -3389,11 +3391,24 @@ static int gaudi2_init_protection_bits(struct hl_device *hdev)
/* PSOC.
* Except for PSOC_GLOBAL_CONF, skip when security is enabled in F/W, because the blocks are
* protected by privileged RR.
+ * For PSOC_GLOBAL_CONF, need to un-secure the scratchpad register which is used for engine
+ * cores to raise events towards F/W.
*/
+ engine_core_intr_reg = (u32) (hdev->asic_prop.engine_core_interrupt_reg_addr - CFG_BASE);
+ if (engine_core_intr_reg >= mmPSOC_GLOBAL_CONF_SCRATCHPAD_0 &&
+ engine_core_intr_reg <= mmPSOC_GLOBAL_CONF_SCRATCHPAD_31) {
+ user_regs_array = &engine_core_intr_reg;
+ user_regs_array_size = 1;
+ } else {
+ dev_err(hdev->dev,
+ "Engine cores register for interrupts (%#x) is not a PSOC scratchpad register\n",
+ engine_core_intr_reg);
+ }
+
rc |= hl_init_pb(hdev, HL_PB_SHARED, HL_PB_NA,
HL_PB_SINGLE_INSTANCE, HL_PB_NA,
gaudi2_pb_psoc_global_conf, ARRAY_SIZE(gaudi2_pb_psoc_global_conf),
- NULL, HL_PB_NA);
+ user_regs_array, user_regs_array_size);
if (!hdev->asic_prop.fw_security_enabled)
rc |= hl_init_pb(hdev, HL_PB_SHARED, HL_PB_NA,
diff --git a/drivers/accel/habanalabs/goya/goya.c b/drivers/accel/habanalabs/goya/goya.c
index 7c685e6075f6..1322cb330c57 100644
--- a/drivers/accel/habanalabs/goya/goya.c
+++ b/drivers/accel/habanalabs/goya/goya.c
@@ -466,7 +466,7 @@ int goya_set_fixed_properties(struct hl_device *hdev)
prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
- strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
+ strscpy_pad(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
CARD_NAME_MAX_LEN);
prop->max_pending_cs = GOYA_MAX_PENDING_CS;
@@ -3358,7 +3358,7 @@ static int goya_pin_memory_before_cs(struct hl_device *hdev,
list_add_tail(&userptr->job_node, parser->job_userptr_list);
- rc = hdev->asic_funcs->asic_dma_map_sgtable(hdev, userptr->sgt, dir);
+ rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir);
if (rc) {
dev_err(hdev->dev, "failed to map sgt with DMA region\n");
goto unpin_memory;
@@ -5122,7 +5122,7 @@ int goya_cpucp_info_get(struct hl_device *hdev)
}
if (!strlen(prop->cpucp_info.card_name))
- strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
+ strscpy_pad(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
CARD_NAME_MAX_LEN);
return 0;
@@ -5465,9 +5465,9 @@ static const struct hl_asic_funcs goya_funcs = {
.asic_dma_pool_free = goya_dma_pool_free,
.cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
.cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
- .hl_dma_unmap_sgtable = hl_dma_unmap_sgtable,
+ .dma_unmap_sgtable = hl_asic_dma_unmap_sgtable,
.cs_parser = goya_cs_parser,
- .asic_dma_map_sgtable = hl_dma_map_sgtable,
+ .dma_map_sgtable = hl_asic_dma_map_sgtable,
.add_end_of_cb_packets = goya_add_end_of_cb_packets,
.update_eq_ci = goya_update_eq_ci,
.context_switch = goya_context_switch,
diff --git a/drivers/accel/habanalabs/goya/goyaP.h b/drivers/accel/habanalabs/goya/goyaP.h
index 5df3d30b91fd..194c2ae157cd 100644
--- a/drivers/accel/habanalabs/goya/goyaP.h
+++ b/drivers/accel/habanalabs/goya/goyaP.h
@@ -9,8 +9,8 @@
#define GOYAP_H_
#include <uapi/drm/habanalabs_accel.h>
+#include <linux/habanalabs/hl_boot_if.h>
#include "../common/habanalabs.h"
-#include "../include/common/hl_boot_if.h"
#include "../include/goya/goya_packets.h"
#include "../include/goya/goya.h"
#include "../include/goya/goya_async_events.h"
diff --git a/drivers/accel/habanalabs/goya/goya_coresight.c b/drivers/accel/habanalabs/goya/goya_coresight.c
index a6d6cc38bcd8..41cae5fd843b 100644
--- a/drivers/accel/habanalabs/goya/goya_coresight.c
+++ b/drivers/accel/habanalabs/goya/goya_coresight.c
@@ -315,6 +315,11 @@ static int goya_config_etf(struct hl_device *hdev,
WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+ val = RREG32(base_reg + 0x20);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
val = RREG32(base_reg + 0x304);
val |= 0x1000;
WREG32(base_reg + 0x304, val);
@@ -386,6 +391,11 @@ static int goya_config_etr(struct hl_device *hdev,
WREG32(mmPSOC_ETR_LAR, CORESIGHT_UNLOCK);
+ val = RREG32(mmPSOC_ETR_CTL);
+
+ if ((!params->enable && val == 0x0) || (params->enable && val != 0x0))
+ return 0;
+
val = RREG32(mmPSOC_ETR_FFCR);
val |= 0x1000;
WREG32(mmPSOC_ETR_FFCR, val);
diff --git a/drivers/accel/habanalabs/include/common/cpucp_if.h b/drivers/accel/habanalabs/include/common/cpucp_if.h
deleted file mode 100644
index 33807b839c37..000000000000
--- a/drivers/accel/habanalabs/include/common/cpucp_if.h
+++ /dev/null
@@ -1,1401 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
- * Copyright 2020-2022 HabanaLabs, Ltd.
- * All Rights Reserved.
- *
- */
-
-#ifndef CPUCP_IF_H
-#define CPUCP_IF_H
-
-#include <linux/types.h>
-#include <linux/if_ether.h>
-
-#include "hl_boot_if.h"
-
-#define NUM_HBM_PSEUDO_CH 2
-#define NUM_HBM_CH_PER_DEV 8
-#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_SHIFT 0
-#define CPUCP_PKT_HBM_ECC_INFO_WR_PAR_MASK 0x00000001
-#define CPUCP_PKT_HBM_ECC_INFO_RD_PAR_SHIFT 1
-#define CPUCP_PKT_HBM_ECC_INFO_RD_PAR_MASK 0x00000002
-#define CPUCP_PKT_HBM_ECC_INFO_CA_PAR_SHIFT 2
-#define CPUCP_PKT_HBM_ECC_INFO_CA_PAR_MASK 0x00000004
-#define CPUCP_PKT_HBM_ECC_INFO_DERR_SHIFT 3
-#define CPUCP_PKT_HBM_ECC_INFO_DERR_MASK 0x00000008
-#define CPUCP_PKT_HBM_ECC_INFO_SERR_SHIFT 4
-#define CPUCP_PKT_HBM_ECC_INFO_SERR_MASK 0x00000010
-#define CPUCP_PKT_HBM_ECC_INFO_TYPE_SHIFT 5
-#define CPUCP_PKT_HBM_ECC_INFO_TYPE_MASK 0x00000020
-#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_SHIFT 6
-#define CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK 0x000007C0
-
-#define PLL_MAP_MAX_BITS 128
-#define PLL_MAP_LEN (PLL_MAP_MAX_BITS / 8)
-
-/*
- * info of the pkt queue pointers in the first async occurrence
- */
-struct cpucp_pkt_sync_err {
- __le32 pi;
- __le32 ci;
-};
-
-struct hl_eq_hbm_ecc_data {
- /* SERR counter */
- __le32 sec_cnt;
- /* DERR counter */
- __le32 dec_cnt;
- /* Supplemental Information according to the mask bits */
- __le32 hbm_ecc_info;
- /* Address in hbm where the ecc happened */
- __le32 first_addr;
- /* SERR continuous address counter */
- __le32 sec_cont_cnt;
- __le32 pad;
-};
-
-/*
- * EVENT QUEUE
- */
-
-struct hl_eq_header {
- __le32 reserved;
- __le32 ctl;
-};
-
-struct hl_eq_ecc_data {
- __le64 ecc_address;
- __le64 ecc_syndrom;
- __u8 memory_wrapper_idx;
- __u8 is_critical;
- __u8 pad[6];
-};
-
-enum hl_sm_sei_cause {
- SM_SEI_SO_OVERFLOW,
- SM_SEI_LBW_4B_UNALIGNED,
- SM_SEI_AXI_RESPONSE_ERR
-};
-
-struct hl_eq_sm_sei_data {
- __le32 sei_log;
- /* enum hl_sm_sei_cause */
- __u8 sei_cause;
- __u8 pad[3];
-};
-
-enum hl_fw_alive_severity {
- FW_ALIVE_SEVERITY_MINOR,
- FW_ALIVE_SEVERITY_CRITICAL
-};
-
-struct hl_eq_fw_alive {
- __le64 uptime_seconds;
- __le32 process_id;
- __le32 thread_id;
- /* enum hl_fw_alive_severity */
- __u8 severity;
- __u8 pad[7];
-};
-
-struct hl_eq_intr_cause {
- __le64 intr_cause_data;
-};
-
-struct hl_eq_pcie_drain_ind_data {
- struct hl_eq_intr_cause intr_cause;
- __le64 drain_wr_addr_lbw;
- __le64 drain_rd_addr_lbw;
- __le64 drain_wr_addr_hbw;
- __le64 drain_rd_addr_hbw;
-};
-
-struct hl_eq_razwi_lbw_info_regs {
- __le32 rr_aw_razwi_reg;
- __le32 rr_aw_razwi_id_reg;
- __le32 rr_ar_razwi_reg;
- __le32 rr_ar_razwi_id_reg;
-};
-
-struct hl_eq_razwi_hbw_info_regs {
- __le32 rr_aw_razwi_hi_reg;
- __le32 rr_aw_razwi_lo_reg;
- __le32 rr_aw_razwi_id_reg;
- __le32 rr_ar_razwi_hi_reg;
- __le32 rr_ar_razwi_lo_reg;
- __le32 rr_ar_razwi_id_reg;
-};
-
-/* razwi_happened masks */
-#define RAZWI_HAPPENED_HBW 0x1
-#define RAZWI_HAPPENED_LBW 0x2
-#define RAZWI_HAPPENED_AW 0x4
-#define RAZWI_HAPPENED_AR 0x8
-
-struct hl_eq_razwi_info {
- __le32 razwi_happened_mask;
- union {
- struct hl_eq_razwi_lbw_info_regs lbw;
- struct hl_eq_razwi_hbw_info_regs hbw;
- };
- __le32 pad;
-};
-
-struct hl_eq_razwi_with_intr_cause {
- struct hl_eq_razwi_info razwi_info;
- struct hl_eq_intr_cause intr_cause;
-};
-
-#define HBM_CA_ERR_CMD_LIFO_LEN 8
-#define HBM_RD_ERR_DATA_LIFO_LEN 8
-#define HBM_WR_PAR_CMD_LIFO_LEN 11
-
-enum hl_hbm_sei_cause {
- /* Command/address parity error event is split into 2 events due to
- * size limitation: ODD suffix for odd HBM CK_t cycles and EVEN suffix
- * for even HBM CK_t cycles
- */
- HBM_SEI_CMD_PARITY_EVEN,
- HBM_SEI_CMD_PARITY_ODD,
- /* Read errors can be reflected as a combination of SERR/DERR/parity
- * errors. Therefore, we define one event for all read error types.
- * LKD will perform further proccessing.
- */
- HBM_SEI_READ_ERR,
- HBM_SEI_WRITE_DATA_PARITY_ERR,
- HBM_SEI_CATTRIP,
- HBM_SEI_MEM_BIST_FAIL,
- HBM_SEI_DFI,
- HBM_SEI_INV_TEMP_READ_OUT,
- HBM_SEI_BIST_FAIL,
-};
-
-/* Masks for parsing hl_hbm_sei_headr fields */
-#define HBM_ECC_SERR_CNTR_MASK 0xFF
-#define HBM_ECC_DERR_CNTR_MASK 0xFF00
-#define HBM_RD_PARITY_CNTR_MASK 0xFF0000
-
-/* HBM index and MC index are known by the event_id */
-struct hl_hbm_sei_header {
- union {
- /* relevant only in case of HBM read error */
- struct {
- __u8 ecc_serr_cnt;
- __u8 ecc_derr_cnt;
- __u8 read_par_cnt;
- __u8 reserved;
- };
- /* All other cases */
- __le32 cnt;
- };
- __u8 sei_cause; /* enum hl_hbm_sei_cause */
- __u8 mc_channel; /* range: 0-3 */
- __u8 mc_pseudo_channel; /* range: 0-7 */
- __u8 is_critical;
-};
-
-#define HBM_RD_ADDR_SID_SHIFT 0
-#define HBM_RD_ADDR_SID_MASK 0x1
-#define HBM_RD_ADDR_BG_SHIFT 1
-#define HBM_RD_ADDR_BG_MASK 0x6
-#define HBM_RD_ADDR_BA_SHIFT 3
-#define HBM_RD_ADDR_BA_MASK 0x18
-#define HBM_RD_ADDR_COL_SHIFT 5
-#define HBM_RD_ADDR_COL_MASK 0x7E0
-#define HBM_RD_ADDR_ROW_SHIFT 11
-#define HBM_RD_ADDR_ROW_MASK 0x3FFF800
-
-struct hbm_rd_addr {
- union {
- /* bit fields are only for FW use */
- struct {
- u32 dbg_rd_err_addr_sid:1;
- u32 dbg_rd_err_addr_bg:2;
- u32 dbg_rd_err_addr_ba:2;
- u32 dbg_rd_err_addr_col:6;
- u32 dbg_rd_err_addr_row:15;
- u32 reserved:6;
- };
- __le32 rd_addr_val;
- };
-};
-
-#define HBM_RD_ERR_BEAT_SHIFT 2
-/* dbg_rd_err_misc fields: */
-/* Read parity is calculated per DW on every beat */
-#define HBM_RD_ERR_PAR_ERR_BEAT0_SHIFT 0
-#define HBM_RD_ERR_PAR_ERR_BEAT0_MASK 0x3
-#define HBM_RD_ERR_PAR_DATA_BEAT0_SHIFT 8
-#define HBM_RD_ERR_PAR_DATA_BEAT0_MASK 0x300
-/* ECC is calculated per PC on every beat */
-#define HBM_RD_ERR_SERR_BEAT0_SHIFT 16
-#define HBM_RD_ERR_SERR_BEAT0_MASK 0x10000
-#define HBM_RD_ERR_DERR_BEAT0_SHIFT 24
-#define HBM_RD_ERR_DERR_BEAT0_MASK 0x100000
-
-struct hl_eq_hbm_sei_read_err_intr_info {
- /* DFI_RD_ERR_REP_ADDR */
- struct hbm_rd_addr dbg_rd_err_addr;
- /* DFI_RD_ERR_REP_ERR */
- union {
- struct {
- /* bit fields are only for FW use */
- u32 dbg_rd_err_par:8;
- u32 dbg_rd_err_par_data:8;
- u32 dbg_rd_err_serr:4;
- u32 dbg_rd_err_derr:4;
- u32 reserved:8;
- };
- __le32 dbg_rd_err_misc;
- };
- /* DFI_RD_ERR_REP_DM */
- __le32 dbg_rd_err_dm;
- /* DFI_RD_ERR_REP_SYNDROME */
- __le32 dbg_rd_err_syndrome;
- /* DFI_RD_ERR_REP_DATA */
- __le32 dbg_rd_err_data[HBM_RD_ERR_DATA_LIFO_LEN];
-};
-
-struct hl_eq_hbm_sei_ca_par_intr_info {
- /* 14 LSBs */
- __le16 dbg_row[HBM_CA_ERR_CMD_LIFO_LEN];
- /* 18 LSBs */
- __le32 dbg_col[HBM_CA_ERR_CMD_LIFO_LEN];
-};
-
-#define WR_PAR_LAST_CMD_COL_SHIFT 0
-#define WR_PAR_LAST_CMD_COL_MASK 0x3F
-#define WR_PAR_LAST_CMD_BG_SHIFT 6
-#define WR_PAR_LAST_CMD_BG_MASK 0xC0
-#define WR_PAR_LAST_CMD_BA_SHIFT 8
-#define WR_PAR_LAST_CMD_BA_MASK 0x300
-#define WR_PAR_LAST_CMD_SID_SHIFT 10
-#define WR_PAR_LAST_CMD_SID_MASK 0x400
-
-/* Row address isn't latched */
-struct hbm_sei_wr_cmd_address {
- /* DFI_DERR_LAST_CMD */
- union {
- struct {
- /* bit fields are only for FW use */
- u32 col:6;
- u32 bg:2;
- u32 ba:2;
- u32 sid:1;
- u32 reserved:21;
- };
- __le32 dbg_wr_cmd_addr;
- };
-};
-
-struct hl_eq_hbm_sei_wr_par_intr_info {
- /* entry 0: WR command address from the 1st cycle prior to the error
- * entry 1: WR command address from the 2nd cycle prior to the error
- * and so on...
- */
- struct hbm_sei_wr_cmd_address dbg_last_wr_cmds[HBM_WR_PAR_CMD_LIFO_LEN];
- /* derr[0:1] - 1st HBM cycle DERR output
- * derr[2:3] - 2nd HBM cycle DERR output
- */
- __u8 dbg_derr;
- /* extend to reach 8B */
- __u8 pad[3];
-};
-
-/*
- * this struct represents the following sei causes:
- * command parity, ECC double error, ECC single error, dfi error, cattrip,
- * temperature read-out, read parity error and write parity error.
- * some only use the header while some have extra data.
- */
-struct hl_eq_hbm_sei_data {
- struct hl_hbm_sei_header hdr;
- union {
- struct hl_eq_hbm_sei_ca_par_intr_info ca_parity_even_info;
- struct hl_eq_hbm_sei_ca_par_intr_info ca_parity_odd_info;
- struct hl_eq_hbm_sei_read_err_intr_info read_err_info;
- struct hl_eq_hbm_sei_wr_par_intr_info wr_parity_info;
- };
-};
-
-/* Engine/farm arc interrupt type */
-enum hl_engine_arc_interrupt_type {
- /* Qman/farm ARC DCCM QUEUE FULL interrupt type */
- ENGINE_ARC_DCCM_QUEUE_FULL_IRQ = 1
-};
-
-/* Data structure specifies details of payload of DCCM QUEUE FULL interrupt */
-struct hl_engine_arc_dccm_queue_full_irq {
- /* Queue index value which caused DCCM QUEUE FULL */
- __le32 queue_index;
- __le32 pad;
-};
-
-/* Data structure specifies details of QM/FARM ARC interrupt */
-struct hl_eq_engine_arc_intr_data {
- /* ARC engine id e.g. DCORE0_TPC0_QM_ARC, DCORE0_TCP1_QM_ARC */
- __le32 engine_id;
- __le32 intr_type; /* enum hl_engine_arc_interrupt_type */
- /* More info related to the interrupt e.g. queue index
- * incase of DCCM_QUEUE_FULL interrupt.
- */
- __le64 payload;
- __le64 pad[5];
-};
-
-#define ADDR_DEC_ADDRESS_COUNT_MAX 4
-
-/* Data structure specifies details of ADDR_DEC interrupt */
-struct hl_eq_addr_dec_intr_data {
- struct hl_eq_intr_cause intr_cause;
- __le64 addr[ADDR_DEC_ADDRESS_COUNT_MAX];
- __u8 addr_cnt;
- __u8 pad[7];
-};
-
-struct hl_eq_entry {
- struct hl_eq_header hdr;
- union {
- __le64 data_placeholder;
- struct hl_eq_ecc_data ecc_data;
- struct hl_eq_hbm_ecc_data hbm_ecc_data; /* Obsolete */
- struct hl_eq_sm_sei_data sm_sei_data;
- struct cpucp_pkt_sync_err pkt_sync_err;
- struct hl_eq_fw_alive fw_alive;
- struct hl_eq_intr_cause intr_cause;
- struct hl_eq_pcie_drain_ind_data pcie_drain_ind_data;
- struct hl_eq_razwi_info razwi_info;
- struct hl_eq_razwi_with_intr_cause razwi_with_intr_cause;
- struct hl_eq_hbm_sei_data sei_data; /* Gaudi2 HBM */
- struct hl_eq_engine_arc_intr_data arc_data;
- struct hl_eq_addr_dec_intr_data addr_dec;
- __le64 data[7];
- };
-};
-
-#define HL_EQ_ENTRY_SIZE sizeof(struct hl_eq_entry)
-
-#define EQ_CTL_READY_SHIFT 31
-#define EQ_CTL_READY_MASK 0x80000000
-
-#define EQ_CTL_EVENT_TYPE_SHIFT 16
-#define EQ_CTL_EVENT_TYPE_MASK 0x0FFF0000
-
-#define EQ_CTL_INDEX_SHIFT 0
-#define EQ_CTL_INDEX_MASK 0x0000FFFF
-
-enum pq_init_status {
- PQ_INIT_STATUS_NA = 0,
- PQ_INIT_STATUS_READY_FOR_CP,
- PQ_INIT_STATUS_READY_FOR_HOST,
- PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI,
- PQ_INIT_STATUS_LEN_NOT_POWER_OF_TWO_ERR,
- PQ_INIT_STATUS_ILLEGAL_Q_ADDR_ERR
-};
-
-/*
- * CpuCP Primary Queue Packets
- *
- * During normal operation, the host's kernel driver needs to send various
- * messages to CpuCP, usually either to SET some value into a H/W periphery or
- * to GET the current value of some H/W periphery. For example, SET the
- * frequency of MME/TPC and GET the value of the thermal sensor.
- *
- * These messages can be initiated either by the User application or by the
- * host's driver itself, e.g. power management code. In either case, the
- * communication from the host's driver to CpuCP will *always* be in
- * synchronous mode, meaning that the host will send a single message and poll
- * until the message was acknowledged and the results are ready (if results are
- * needed).
- *
- * This means that only a single message can be sent at a time and the host's
- * driver must wait for its result before sending the next message. Having said
- * that, because these are control messages which are sent in a relatively low
- * frequency, this limitation seems acceptable. It's important to note that
- * in case of multiple devices, messages to different devices *can* be sent
- * at the same time.
- *
- * The message, inputs/outputs (if relevant) and fence object will be located
- * on the device DDR at an address that will be determined by the host's driver.
- * During device initialization phase, the host will pass to CpuCP that address.
- * Most of the message types will contain inputs/outputs inside the message
- * itself. The common part of each message will contain the opcode of the
- * message (its type) and a field representing a fence object.
- *
- * When the host's driver wishes to send a message to CPU CP, it will write the
- * message contents to the device DDR, clear the fence object and then write to
- * the PSOC_ARC1_AUX_SW_INTR, to issue interrupt 121 to ARC Management CPU.
- *
- * Upon receiving the interrupt (#121), CpuCP will read the message from the
- * DDR. In case the message is a SET operation, CpuCP will first perform the
- * operation and then write to the fence object on the device DDR. In case the
- * message is a GET operation, CpuCP will first fill the results section on the
- * device DDR and then write to the fence object. If an error occurred, CpuCP
- * will fill the rc field with the right error code.
- *
- * In the meantime, the host's driver will poll on the fence object. Once the
- * host sees that the fence object is signaled, it will read the results from
- * the device DDR (if relevant) and resume the code execution in the host's
- * driver.
- *
- * To use QMAN packets, the opcode must be the QMAN opcode, shifted by 8
- * so the value being put by the host's driver matches the value read by CpuCP
- *
- * Non-QMAN packets should be limited to values 1 through (2^8 - 1)
- *
- * Detailed description:
- *
- * CPUCP_PACKET_DISABLE_PCI_ACCESS -
- * After receiving this packet the embedded CPU must NOT issue PCI
- * transactions (read/write) towards the Host CPU. This also include
- * sending MSI-X interrupts.
- * This packet is usually sent before the device is moved to D3Hot state.
- *
- * CPUCP_PACKET_ENABLE_PCI_ACCESS -
- * After receiving this packet the embedded CPU is allowed to issue PCI
- * transactions towards the Host CPU, including sending MSI-X interrupts.
- * This packet is usually send after the device is moved to D0 state.
- *
- * CPUCP_PACKET_TEMPERATURE_GET -
- * Fetch the current temperature / Max / Max Hyst / Critical /
- * Critical Hyst of a specified thermal sensor. The packet's
- * arguments specify the desired sensor and the field to get.
- *
- * CPUCP_PACKET_VOLTAGE_GET -
- * Fetch the voltage / Max / Min of a specified sensor. The packet's
- * arguments specify the sensor and type.
- *
- * CPUCP_PACKET_CURRENT_GET -
- * Fetch the current / Max / Min of a specified sensor. The packet's
- * arguments specify the sensor and type.
- *
- * CPUCP_PACKET_FAN_SPEED_GET -
- * Fetch the speed / Max / Min of a specified fan. The packet's
- * arguments specify the sensor and type.
- *
- * CPUCP_PACKET_PWM_GET -
- * Fetch the pwm value / mode of a specified pwm. The packet's
- * arguments specify the sensor and type.
- *
- * CPUCP_PACKET_PWM_SET -
- * Set the pwm value / mode of a specified pwm. The packet's
- * arguments specify the sensor, type and value.
- *
- * CPUCP_PACKET_FREQUENCY_SET -
- * Set the frequency of a specified PLL. The packet's arguments specify
- * the PLL and the desired frequency. The actual frequency in the device
- * might differ from the requested frequency.
- *
- * CPUCP_PACKET_FREQUENCY_GET -
- * Fetch the frequency of a specified PLL. The packet's arguments specify
- * the PLL.
- *
- * CPUCP_PACKET_LED_SET -
- * Set the state of a specified led. The packet's arguments
- * specify the led and the desired state.
- *
- * CPUCP_PACKET_I2C_WR -
- * Write 32-bit value to I2C device. The packet's arguments specify the
- * I2C bus, address and value.
- *
- * CPUCP_PACKET_I2C_RD -
- * Read 32-bit value from I2C device. The packet's arguments specify the
- * I2C bus and address.
- *
- * CPUCP_PACKET_INFO_GET -
- * Fetch information from the device as specified in the packet's
- * structure. The host's driver passes the max size it allows the CpuCP to
- * write to the structure, to prevent data corruption in case of
- * mismatched driver/FW versions.
- *
- * CPUCP_PACKET_FLASH_PROGRAM_REMOVED - this packet was removed
- *
- * CPUCP_PACKET_UNMASK_RAZWI_IRQ -
- * Unmask the given IRQ. The IRQ number is specified in the value field.
- * The packet is sent after receiving an interrupt and printing its
- * relevant information.
- *
- * CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY -
- * Unmask the given IRQs. The IRQs numbers are specified in an array right
- * after the cpucp_packet structure, where its first element is the array
- * length. The packet is sent after a soft reset was done in order to
- * handle any interrupts that were sent during the reset process.
- *
- * CPUCP_PACKET_TEST -
- * Test packet for CpuCP connectivity. The CPU will put the fence value
- * in the result field.
- *
- * CPUCP_PACKET_FREQUENCY_CURR_GET -
- * Fetch the current frequency of a specified PLL. The packet's arguments
- * specify the PLL.
- *
- * CPUCP_PACKET_MAX_POWER_GET -
- * Fetch the maximal power of the device.
- *
- * CPUCP_PACKET_MAX_POWER_SET -
- * Set the maximal power of the device. The packet's arguments specify
- * the power.
- *
- * CPUCP_PACKET_EEPROM_DATA_GET -
- * Get EEPROM data from the CpuCP kernel. The buffer is specified in the
- * addr field. The CPU will put the returned data size in the result
- * field. In addition, the host's driver passes the max size it allows the
- * CpuCP to write to the structure, to prevent data corruption in case of
- * mismatched driver/FW versions.
- *
- * CPUCP_PACKET_NIC_INFO_GET -
- * Fetch information from the device regarding the NIC. the host's driver
- * passes the max size it allows the CpuCP to write to the structure, to
- * prevent data corruption in case of mismatched driver/FW versions.
- *
- * CPUCP_PACKET_TEMPERATURE_SET -
- * Set the value of the offset property of a specified thermal sensor.
- * The packet's arguments specify the desired sensor and the field to
- * set.
- *
- * CPUCP_PACKET_VOLTAGE_SET -
- * Trigger the reset_history property of a specified voltage sensor.
- * The packet's arguments specify the desired sensor and the field to
- * set.
- *
- * CPUCP_PACKET_CURRENT_SET -
- * Trigger the reset_history property of a specified current sensor.
- * The packet's arguments specify the desired sensor and the field to
- * set.
- *
- * CPUCP_PACKET_PCIE_THROUGHPUT_GET -
- * Get throughput of PCIe.
- * The packet's arguments specify the transaction direction (TX/RX).
- * The window measurement is 10[msec], and the return value is in KB/sec.
- *
- * CPUCP_PACKET_PCIE_REPLAY_CNT_GET
- * Replay count measures number of "replay" events, which is basicly
- * number of retries done by PCIe.
- *
- * CPUCP_PACKET_TOTAL_ENERGY_GET -
- * Total Energy is measurement of energy from the time FW Linux
- * is loaded. It is calculated by multiplying the average power
- * by time (passed from armcp start). The units are in MilliJouls.
- *
- * CPUCP_PACKET_PLL_INFO_GET -
- * Fetch frequencies of PLL from the required PLL IP.
- * The packet's arguments specify the device PLL type
- * Pll type is the PLL from device pll_index enum.
- * The result is composed of 4 outputs, each is 16-bit
- * frequency in MHz.
- *
- * CPUCP_PACKET_POWER_GET -
- * Fetch the present power consumption of the device (Current * Voltage).
- *
- * CPUCP_PACKET_NIC_PFC_SET -
- * Enable/Disable the NIC PFC feature. The packet's arguments specify the
- * NIC port, relevant lanes to configure and one bit indication for
- * enable/disable.
- *
- * CPUCP_PACKET_NIC_FAULT_GET -
- * Fetch the current indication for local/remote faults from the NIC MAC.
- * The result is 32-bit value of the relevant register.
- *
- * CPUCP_PACKET_NIC_LPBK_SET -
- * Enable/Disable the MAC loopback feature. The packet's arguments specify
- * the NIC port, relevant lanes to configure and one bit indication for
- * enable/disable.
- *
- * CPUCP_PACKET_NIC_MAC_INIT -
- * Configure the NIC MAC channels. The packet's arguments specify the
- * NIC port and the speed.
- *
- * CPUCP_PACKET_MSI_INFO_SET -
- * set the index number for each supported msi type going from
- * host to device
- *
- * CPUCP_PACKET_NIC_XPCS91_REGS_GET -
- * Fetch the un/correctable counters values from the NIC MAC.
- *
- * CPUCP_PACKET_NIC_STAT_REGS_GET -
- * Fetch various NIC MAC counters from the NIC STAT.
- *
- * CPUCP_PACKET_NIC_STAT_REGS_CLR -
- * Clear the various NIC MAC counters in the NIC STAT.
- *
- * CPUCP_PACKET_NIC_STAT_REGS_ALL_GET -
- * Fetch all NIC MAC counters from the NIC STAT.
- *
- * CPUCP_PACKET_IS_IDLE_CHECK -
- * Check if the device is IDLE in regard to the DMA/compute engines
- * and QMANs. The f/w will return a bitmask where each bit represents
- * a different engine or QMAN according to enum cpucp_idle_mask.
- * The bit will be 1 if the engine is NOT idle.
- *
- * CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET -
- * Fetch all HBM replaced-rows and prending to be replaced rows data.
- *
- * CPUCP_PACKET_HBM_PENDING_ROWS_STATUS -
- * Fetch status of HBM rows pending replacement and need a reboot to
- * be replaced.
- *
- * CPUCP_PACKET_POWER_SET -
- * Resets power history of device to 0
- *
- * CPUCP_PACKET_ENGINE_CORE_ASID_SET -
- * Packet to perform engine core ASID configuration
- *
- * CPUCP_PACKET_SEC_ATTEST_GET -
- * Get the attestaion data that is collected during various stages of the
- * boot sequence. the attestation data is also hashed with some unique
- * number (nonce) provided by the host to prevent replay attacks.
- * public key and certificate also provided as part of the FW response.
- *
- * CPUCP_PACKET_MONITOR_DUMP_GET -
- * Get monitors registers dump from the CpuCP kernel.
- * The CPU will put the registers dump in the a buffer allocated by the driver
- * which address is passed via the CpuCp packet. In addition, the host's driver
- * passes the max size it allows the CpuCP to write to the structure, to prevent
- * data corruption in case of mismatched driver/FW versions.
- * Obsolete.
- *
- * CPUCP_PACKET_GENERIC_PASSTHROUGH -
- * Generic opcode for all firmware info that is only passed to host
- * through the LKD, without getting parsed there.
- *
- * CPUCP_PACKET_ACTIVE_STATUS_SET -
- * LKD sends FW indication whether device is free or in use, this indication is reported
- * also to the BMC.
- *
- * CPUCP_PACKET_REGISTER_INTERRUPTS -
- * Packet to register interrupts indicating LKD is ready to receive events from FW.
- *
- * CPUCP_PACKET_SOFT_RESET -
- * Packet to perform soft-reset.
- */
-
-enum cpucp_packet_id {
- CPUCP_PACKET_DISABLE_PCI_ACCESS = 1, /* internal */
- CPUCP_PACKET_ENABLE_PCI_ACCESS, /* internal */
- CPUCP_PACKET_TEMPERATURE_GET, /* sysfs */
- CPUCP_PACKET_VOLTAGE_GET, /* sysfs */
- CPUCP_PACKET_CURRENT_GET, /* sysfs */
- CPUCP_PACKET_FAN_SPEED_GET, /* sysfs */
- CPUCP_PACKET_PWM_GET, /* sysfs */
- CPUCP_PACKET_PWM_SET, /* sysfs */
- CPUCP_PACKET_FREQUENCY_SET, /* sysfs */
- CPUCP_PACKET_FREQUENCY_GET, /* sysfs */
- CPUCP_PACKET_LED_SET, /* debugfs */
- CPUCP_PACKET_I2C_WR, /* debugfs */
- CPUCP_PACKET_I2C_RD, /* debugfs */
- CPUCP_PACKET_INFO_GET, /* IOCTL */
- CPUCP_PACKET_FLASH_PROGRAM_REMOVED,
- CPUCP_PACKET_UNMASK_RAZWI_IRQ, /* internal */
- CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY, /* internal */
- CPUCP_PACKET_TEST, /* internal */
- CPUCP_PACKET_FREQUENCY_CURR_GET, /* sysfs */
- CPUCP_PACKET_MAX_POWER_GET, /* sysfs */
- CPUCP_PACKET_MAX_POWER_SET, /* sysfs */
- CPUCP_PACKET_EEPROM_DATA_GET, /* sysfs */
- CPUCP_PACKET_NIC_INFO_GET, /* internal */
- CPUCP_PACKET_TEMPERATURE_SET, /* sysfs */
- CPUCP_PACKET_VOLTAGE_SET, /* sysfs */
- CPUCP_PACKET_CURRENT_SET, /* sysfs */
- CPUCP_PACKET_PCIE_THROUGHPUT_GET, /* internal */
- CPUCP_PACKET_PCIE_REPLAY_CNT_GET, /* internal */
- CPUCP_PACKET_TOTAL_ENERGY_GET, /* internal */
- CPUCP_PACKET_PLL_INFO_GET, /* internal */
- CPUCP_PACKET_NIC_STATUS, /* internal */
- CPUCP_PACKET_POWER_GET, /* internal */
- CPUCP_PACKET_NIC_PFC_SET, /* internal */
- CPUCP_PACKET_NIC_FAULT_GET, /* internal */
- CPUCP_PACKET_NIC_LPBK_SET, /* internal */
- CPUCP_PACKET_NIC_MAC_CFG, /* internal */
- CPUCP_PACKET_MSI_INFO_SET, /* internal */
- CPUCP_PACKET_NIC_XPCS91_REGS_GET, /* internal */
- CPUCP_PACKET_NIC_STAT_REGS_GET, /* internal */
- CPUCP_PACKET_NIC_STAT_REGS_CLR, /* internal */
- CPUCP_PACKET_NIC_STAT_REGS_ALL_GET, /* internal */
- CPUCP_PACKET_IS_IDLE_CHECK, /* internal */
- CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET,/* internal */
- CPUCP_PACKET_HBM_PENDING_ROWS_STATUS, /* internal */
- CPUCP_PACKET_POWER_SET, /* internal */
- CPUCP_PACKET_RESERVED, /* not used */
- CPUCP_PACKET_ENGINE_CORE_ASID_SET, /* internal */
- CPUCP_PACKET_RESERVED2, /* not used */
- CPUCP_PACKET_SEC_ATTEST_GET, /* internal */
- CPUCP_PACKET_RESERVED3, /* not used */
- CPUCP_PACKET_RESERVED4, /* not used */
- CPUCP_PACKET_MONITOR_DUMP_GET, /* debugfs */
- CPUCP_PACKET_RESERVED5, /* not used */
- CPUCP_PACKET_RESERVED6, /* not used */
- CPUCP_PACKET_RESERVED7, /* not used */
- CPUCP_PACKET_GENERIC_PASSTHROUGH, /* IOCTL */
- CPUCP_PACKET_RESERVED8, /* not used */
- CPUCP_PACKET_ACTIVE_STATUS_SET, /* internal */
- CPUCP_PACKET_RESERVED9, /* not used */
- CPUCP_PACKET_RESERVED10, /* not used */
- CPUCP_PACKET_RESERVED11, /* not used */
- CPUCP_PACKET_RESERVED12, /* internal */
- CPUCP_PACKET_REGISTER_INTERRUPTS, /* internal */
- CPUCP_PACKET_SOFT_RESET, /* internal */
- CPUCP_PACKET_ID_MAX /* must be last */
-};
-
-#define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5
-
-#define CPUCP_PKT_CTL_RC_SHIFT 12
-#define CPUCP_PKT_CTL_RC_MASK 0x0000F000
-
-#define CPUCP_PKT_CTL_OPCODE_SHIFT 16
-#define CPUCP_PKT_CTL_OPCODE_MASK 0x1FFF0000
-
-#define CPUCP_PKT_RES_PLL_OUT0_SHIFT 0
-#define CPUCP_PKT_RES_PLL_OUT0_MASK 0x000000000000FFFFull
-#define CPUCP_PKT_RES_PLL_OUT1_SHIFT 16
-#define CPUCP_PKT_RES_PLL_OUT1_MASK 0x00000000FFFF0000ull
-#define CPUCP_PKT_RES_PLL_OUT2_SHIFT 32
-#define CPUCP_PKT_RES_PLL_OUT2_MASK 0x0000FFFF00000000ull
-#define CPUCP_PKT_RES_PLL_OUT3_SHIFT 48
-#define CPUCP_PKT_RES_PLL_OUT3_MASK 0xFFFF000000000000ull
-
-#define CPUCP_PKT_RES_EEPROM_OUT0_SHIFT 0
-#define CPUCP_PKT_RES_EEPROM_OUT0_MASK 0x000000000000FFFFull
-#define CPUCP_PKT_RES_EEPROM_OUT1_SHIFT 16
-#define CPUCP_PKT_RES_EEPROM_OUT1_MASK 0x0000000000FF0000ull
-
-#define CPUCP_PKT_VAL_PFC_IN1_SHIFT 0
-#define CPUCP_PKT_VAL_PFC_IN1_MASK 0x0000000000000001ull
-#define CPUCP_PKT_VAL_PFC_IN2_SHIFT 1
-#define CPUCP_PKT_VAL_PFC_IN2_MASK 0x000000000000001Eull
-
-#define CPUCP_PKT_VAL_LPBK_IN1_SHIFT 0
-#define CPUCP_PKT_VAL_LPBK_IN1_MASK 0x0000000000000001ull
-#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1
-#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull
-
-#define CPUCP_PKT_VAL_MAC_CNT_IN1_SHIFT 0
-#define CPUCP_PKT_VAL_MAC_CNT_IN1_MASK 0x0000000000000001ull
-#define CPUCP_PKT_VAL_MAC_CNT_IN2_SHIFT 1
-#define CPUCP_PKT_VAL_MAC_CNT_IN2_MASK 0x00000000FFFFFFFEull
-
-/* heartbeat status bits */
-#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0
-#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001
-
-struct cpucp_packet {
- union {
- __le64 value; /* For SET packets */
- __le64 result; /* For GET packets */
- __le64 addr; /* For PQ */
- };
-
- __le32 ctl;
-
- __le32 fence; /* Signal to host that message is completed */
-
- union {
- struct {/* For temperature/current/voltage/fan/pwm get/set */
- __le16 sensor_index;
- __le16 type;
- };
-
- struct { /* For I2C read/write */
- __u8 i2c_bus;
- __u8 i2c_addr;
- __u8 i2c_reg;
- /*
- * In legacy implemetations, i2c_len was not present,
- * was unused and just added as pad.
- * So if i2c_len is 0, it is treated as legacy
- * and r/w 1 Byte, else if i2c_len is specified,
- * its treated as new multibyte r/w support.
- */
- __u8 i2c_len;
- };
-
- struct {/* For PLL info fetch */
- __le16 pll_type;
- /* TODO pll_reg is kept temporary before removal */
- __le16 pll_reg;
- };
-
- /* For any general request */
- __le32 index;
-
- /* For frequency get/set */
- __le32 pll_index;
-
- /* For led set */
- __le32 led_index;
-
- /* For get CpuCP info/EEPROM data/NIC info */
- __le32 data_max_size;
-
- /*
- * For any general status bitmask. Shall be used whenever the
- * result cannot be used to hold general purpose data.
- */
- __le32 status_mask;
-
- /* random, used once number, for security packets */
- __le32 nonce;
- };
-
- union {
- /* For NIC requests */
- __le32 port_index;
-
- /* For Generic packet sub index */
- __le32 pkt_subidx;
- };
-};
-
-struct cpucp_unmask_irq_arr_packet {
- struct cpucp_packet cpucp_pkt;
- __le32 length;
- __le32 irqs[];
-};
-
-struct cpucp_nic_status_packet {
- struct cpucp_packet cpucp_pkt;
- __le32 length;
- __le32 data[];
-};
-
-struct cpucp_array_data_packet {
- struct cpucp_packet cpucp_pkt;
- __le32 length;
- __le32 data[];
-};
-
-enum cpucp_led_index {
- CPUCP_LED0_INDEX = 0,
- CPUCP_LED1_INDEX,
- CPUCP_LED2_INDEX,
- CPUCP_LED_MAX_INDEX = CPUCP_LED2_INDEX
-};
-
-/*
- * enum cpucp_packet_rc - Error return code
- * @cpucp_packet_success -> in case of success.
- * @cpucp_packet_invalid -> this is to support first generation platforms.
- * @cpucp_packet_fault -> in case of processing error like failing to
- * get device binding or semaphore etc.
- * @cpucp_packet_invalid_pkt -> when cpucp packet is un-supported.
- * @cpucp_packet_invalid_params -> when checking parameter like length of buffer
- * or attribute value etc.
- * @cpucp_packet_rc_max -> It indicates size of enum so should be at last.
- */
-enum cpucp_packet_rc {
- cpucp_packet_success,
- cpucp_packet_invalid,
- cpucp_packet_fault,
- cpucp_packet_invalid_pkt,
- cpucp_packet_invalid_params,
- cpucp_packet_rc_max
-};
-
-/*
- * cpucp_temp_type should adhere to hwmon_temp_attributes
- * defined in Linux kernel hwmon.h file
- */
-enum cpucp_temp_type {
- cpucp_temp_input,
- cpucp_temp_min = 4,
- cpucp_temp_min_hyst,
- cpucp_temp_max = 6,
- cpucp_temp_max_hyst,
- cpucp_temp_crit,
- cpucp_temp_crit_hyst,
- cpucp_temp_offset = 19,
- cpucp_temp_lowest = 21,
- cpucp_temp_highest = 22,
- cpucp_temp_reset_history = 23,
- cpucp_temp_warn = 24,
- cpucp_temp_max_crit = 25,
- cpucp_temp_max_warn = 26,
-};
-
-enum cpucp_in_attributes {
- cpucp_in_input,
- cpucp_in_min,
- cpucp_in_max,
- cpucp_in_lowest = 6,
- cpucp_in_highest = 7,
- cpucp_in_reset_history,
- cpucp_in_intr_alarm_a,
- cpucp_in_intr_alarm_b,
-};
-
-enum cpucp_curr_attributes {
- cpucp_curr_input,
- cpucp_curr_min,
- cpucp_curr_max,
- cpucp_curr_lowest = 6,
- cpucp_curr_highest = 7,
- cpucp_curr_reset_history
-};
-
-enum cpucp_fan_attributes {
- cpucp_fan_input,
- cpucp_fan_min = 2,
- cpucp_fan_max
-};
-
-enum cpucp_pwm_attributes {
- cpucp_pwm_input,
- cpucp_pwm_enable
-};
-
-enum cpucp_pcie_throughput_attributes {
- cpucp_pcie_throughput_tx,
- cpucp_pcie_throughput_rx
-};
-
-/* TODO temporary kept before removal */
-enum cpucp_pll_reg_attributes {
- cpucp_pll_nr_reg,
- cpucp_pll_nf_reg,
- cpucp_pll_od_reg,
- cpucp_pll_div_factor_reg,
- cpucp_pll_div_sel_reg
-};
-
-/* TODO temporary kept before removal */
-enum cpucp_pll_type_attributes {
- cpucp_pll_cpu,
- cpucp_pll_pci,
-};
-
-/*
- * cpucp_power_type aligns with hwmon_power_attributes
- * defined in Linux kernel hwmon.h file
- */
-enum cpucp_power_type {
- CPUCP_POWER_INPUT = 8,
- CPUCP_POWER_INPUT_HIGHEST = 9,
- CPUCP_POWER_RESET_INPUT_HISTORY = 11
-};
-
-/*
- * MSI type enumeration table for all ASICs and future SW versions.
- * For future ASIC-LKD compatibility, we can only add new enumerations.
- * at the end of the table (before CPUCP_NUM_OF_MSI_TYPES).
- * Changing the order of entries or removing entries is not allowed.
- */
-enum cpucp_msi_type {
- CPUCP_EVENT_QUEUE_MSI_TYPE,
- CPUCP_NIC_PORT1_MSI_TYPE,
- CPUCP_NIC_PORT3_MSI_TYPE,
- CPUCP_NIC_PORT5_MSI_TYPE,
- CPUCP_NIC_PORT7_MSI_TYPE,
- CPUCP_NIC_PORT9_MSI_TYPE,
- CPUCP_NUM_OF_MSI_TYPES
-};
-
-/*
- * PLL enumeration table used for all ASICs and future SW versions.
- * For future ASIC-LKD compatibility, we can only add new enumerations.
- * at the end of the table.
- * Changing the order of entries or removing entries is not allowed.
- */
-enum pll_index {
- CPU_PLL = 0,
- PCI_PLL = 1,
- NIC_PLL = 2,
- DMA_PLL = 3,
- MESH_PLL = 4,
- MME_PLL = 5,
- TPC_PLL = 6,
- IF_PLL = 7,
- SRAM_PLL = 8,
- NS_PLL = 9,
- HBM_PLL = 10,
- MSS_PLL = 11,
- DDR_PLL = 12,
- VID_PLL = 13,
- BANK_PLL = 14,
- MMU_PLL = 15,
- IC_PLL = 16,
- MC_PLL = 17,
- EMMC_PLL = 18,
- D2D_PLL = 19,
- CS_PLL = 20,
- C2C_PLL = 21,
- NCH_PLL = 22,
- C2M_PLL = 23,
- PLL_MAX
-};
-
-enum rl_index {
- TPC_RL = 0,
- MME_RL,
- EDMA_RL,
-};
-
-enum pvt_index {
- PVT_SW,
- PVT_SE,
- PVT_NW,
- PVT_NE
-};
-
-/* Event Queue Packets */
-
-struct eq_generic_event {
- __le64 data[7];
-};
-
-/*
- * CpuCP info
- */
-
-#define CARD_NAME_MAX_LEN 16
-#define CPUCP_MAX_SENSORS 128
-#define CPUCP_MAX_NICS 128
-#define CPUCP_LANES_PER_NIC 4
-#define CPUCP_NIC_QSFP_EEPROM_MAX_LEN 1024
-#define CPUCP_MAX_NIC_LANES (CPUCP_MAX_NICS * CPUCP_LANES_PER_NIC)
-#define CPUCP_NIC_MASK_ARR_LEN ((CPUCP_MAX_NICS + 63) / 64)
-#define CPUCP_NIC_POLARITY_ARR_LEN ((CPUCP_MAX_NIC_LANES + 63) / 64)
-#define CPUCP_HBM_ROW_REPLACE_MAX 32
-
-struct cpucp_sensor {
- __le32 type;
- __le32 flags;
-};
-
-/**
- * struct cpucp_card_types - ASIC card type.
- * @cpucp_card_type_pci: PCI card.
- * @cpucp_card_type_pmc: PCI Mezzanine Card.
- */
-enum cpucp_card_types {
- cpucp_card_type_pci,
- cpucp_card_type_pmc
-};
-
-#define CPUCP_SEC_CONF_ENABLED_SHIFT 0
-#define CPUCP_SEC_CONF_ENABLED_MASK 0x00000001
-
-#define CPUCP_SEC_CONF_FLASH_WP_SHIFT 1
-#define CPUCP_SEC_CONF_FLASH_WP_MASK 0x00000002
-
-#define CPUCP_SEC_CONF_EEPROM_WP_SHIFT 2
-#define CPUCP_SEC_CONF_EEPROM_WP_MASK 0x00000004
-
-/**
- * struct cpucp_security_info - Security information.
- * @config: configuration bit field
- * @keys_num: number of stored keys
- * @revoked_keys: revoked keys bit field
- * @min_svn: minimal security version
- */
-struct cpucp_security_info {
- __u8 config;
- __u8 keys_num;
- __u8 revoked_keys;
- __u8 min_svn;
-};
-
-/**
- * struct cpucp_info - Info from CpuCP that is necessary to the host's driver
- * @sensors: available sensors description.
- * @kernel_version: CpuCP linux kernel version.
- * @reserved: reserved field.
- * @card_type: card configuration type.
- * @card_location: in a server, each card has different connections topology
- * depending on its location (relevant for PMC card type)
- * @cpld_version: CPLD programmed F/W version.
- * @infineon_version: Infineon main DC-DC version.
- * @fuse_version: silicon production FUSE information.
- * @thermal_version: thermald S/W version.
- * @cpucp_version: CpuCP S/W version.
- * @infineon_second_stage_version: Infineon 2nd stage DC-DC version.
- * @dram_size: available DRAM size.
- * @card_name: card name that will be displayed in HWMON subsystem on the host
- * @tpc_binning_mask: TPC binning mask, 1 bit per TPC instance
- * (0 = functional, 1 = binned)
- * @decoder_binning_mask: Decoder binning mask, 1 bit per decoder instance
- * (0 = functional, 1 = binned), maximum 1 per dcore
- * @sram_binning: Categorize SRAM functionality
- * (0 = fully functional, 1 = lower-half is not functional,
- * 2 = upper-half is not functional)
- * @sec_info: security information
- * @pll_map: Bit map of supported PLLs for current ASIC version.
- * @mme_binning_mask: MME binning mask,
- * bits [0:6] <==> dcore0 mme fma
- * bits [7:13] <==> dcore1 mme fma
- * bits [14:20] <==> dcore0 mme ima
- * bits [21:27] <==> dcore1 mme ima
- * For each group, if the 6th bit is set then first 5 bits
- * represent the col's idx [0-31], otherwise these bits are
- * ignored, and col idx 32 is binned. 7th bit is don't care.
- * @dram_binning_mask: DRAM binning mask, 1 bit per dram instance
- * (0 = functional 1 = binned)
- * @memory_repair_flag: eFuse flag indicating memory repair
- * @edma_binning_mask: EDMA binning mask, 1 bit per EDMA instance
- * (0 = functional 1 = binned)
- * @xbar_binning_mask: Xbar binning mask, 1 bit per Xbar instance
- * (0 = functional 1 = binned)
- * @interposer_version: Interposer version programmed in eFuse
- * @substrate_version: Substrate version programmed in eFuse
- * @fw_hbm_region_size: Size in bytes of FW reserved region in HBM.
- * @fw_os_version: Firmware OS Version
- */
-struct cpucp_info {
- struct cpucp_sensor sensors[CPUCP_MAX_SENSORS];
- __u8 kernel_version[VERSION_MAX_LEN];
- __le32 reserved;
- __le32 card_type;
- __le32 card_location;
- __le32 cpld_version;
- __le32 infineon_version;
- __u8 fuse_version[VERSION_MAX_LEN];
- __u8 thermal_version[VERSION_MAX_LEN];
- __u8 cpucp_version[VERSION_MAX_LEN];
- __le32 infineon_second_stage_version;
- __le64 dram_size;
- char card_name[CARD_NAME_MAX_LEN];
- __le64 tpc_binning_mask;
- __le64 decoder_binning_mask;
- __u8 sram_binning;
- __u8 dram_binning_mask;
- __u8 memory_repair_flag;
- __u8 edma_binning_mask;
- __u8 xbar_binning_mask;
- __u8 interposer_version;
- __u8 substrate_version;
- __u8 reserved2;
- struct cpucp_security_info sec_info;
- __le32 fw_hbm_region_size;
- __u8 pll_map[PLL_MAP_LEN];
- __le64 mme_binning_mask;
- __u8 fw_os_version[VERSION_MAX_LEN];
-};
-
-struct cpucp_mac_addr {
- __u8 mac_addr[ETH_ALEN];
-};
-
-enum cpucp_serdes_type {
- TYPE_1_SERDES_TYPE,
- TYPE_2_SERDES_TYPE,
- HLS1_SERDES_TYPE,
- HLS1H_SERDES_TYPE,
- HLS2_SERDES_TYPE,
- HLS2_TYPE_1_SERDES_TYPE,
- MAX_NUM_SERDES_TYPE, /* number of types */
- UNKNOWN_SERDES_TYPE = 0xFFFF /* serdes_type is u16 */
-};
-
-struct cpucp_nic_info {
- struct cpucp_mac_addr mac_addrs[CPUCP_MAX_NICS];
- __le64 link_mask[CPUCP_NIC_MASK_ARR_LEN];
- __le64 pol_tx_mask[CPUCP_NIC_POLARITY_ARR_LEN];
- __le64 pol_rx_mask[CPUCP_NIC_POLARITY_ARR_LEN];
- __le64 link_ext_mask[CPUCP_NIC_MASK_ARR_LEN];
- __u8 qsfp_eeprom[CPUCP_NIC_QSFP_EEPROM_MAX_LEN];
- __le64 auto_neg_mask[CPUCP_NIC_MASK_ARR_LEN];
- __le16 serdes_type; /* enum cpucp_serdes_type */
- __le16 tx_swap_map[CPUCP_MAX_NICS];
- __u8 reserved[6];
-};
-
-#define PAGE_DISCARD_MAX 64
-
-struct page_discard_info {
- __u8 num_entries;
- __u8 reserved[7];
- __le32 mmu_page_idx[PAGE_DISCARD_MAX];
-};
-
-/*
- * struct frac_val - fracture value represented by "integer.frac".
- * @integer: the integer part of the fracture value;
- * @frac: the fracture part of the fracture value.
- */
-struct frac_val {
- union {
- struct {
- __le16 integer;
- __le16 frac;
- };
- __le32 val;
- };
-};
-
-/*
- * struct ser_val - the SER (symbol error rate) value is represented by "integer * 10 ^ -exp".
- * @integer: the integer part of the SER value;
- * @exp: the exponent part of the SER value.
- */
-struct ser_val {
- __le16 integer;
- __le16 exp;
-};
-
-/*
- * struct cpucp_nic_status - describes the status of a NIC port.
- * @port: NIC port index.
- * @bad_format_cnt: e.g. CRC.
- * @responder_out_of_sequence_psn_cnt: e.g NAK.
- * @high_ber_reinit_cnt: link reinit due to high BER.
- * @correctable_err_cnt: e.g. bit-flip.
- * @uncorrectable_err_cnt: e.g. MAC errors.
- * @retraining_cnt: re-training counter.
- * @up: is port up.
- * @pcs_link: has PCS link.
- * @phy_ready: is PHY ready.
- * @auto_neg: is Autoneg enabled.
- * @timeout_retransmission_cnt: timeout retransmission events.
- * @high_ber_cnt: high ber events.
- * @pre_fec_ser: pre FEC SER value.
- * @post_fec_ser: post FEC SER value.
- * @throughput: measured throughput.
- * @latency: measured latency.
- */
-struct cpucp_nic_status {
- __le32 port;
- __le32 bad_format_cnt;
- __le32 responder_out_of_sequence_psn_cnt;
- __le32 high_ber_reinit;
- __le32 correctable_err_cnt;
- __le32 uncorrectable_err_cnt;
- __le32 retraining_cnt;
- __u8 up;
- __u8 pcs_link;
- __u8 phy_ready;
- __u8 auto_neg;
- __le32 timeout_retransmission_cnt;
- __le32 high_ber_cnt;
- struct ser_val pre_fec_ser;
- struct ser_val post_fec_ser;
- struct frac_val bandwidth;
- struct frac_val lat;
-};
-
-enum cpucp_hbm_row_replace_cause {
- REPLACE_CAUSE_DOUBLE_ECC_ERR,
- REPLACE_CAUSE_MULTI_SINGLE_ECC_ERR,
-};
-
-struct cpucp_hbm_row_info {
- __u8 hbm_idx;
- __u8 pc;
- __u8 sid;
- __u8 bank_idx;
- __le16 row_addr;
- __u8 replaced_row_cause; /* enum cpucp_hbm_row_replace_cause */
- __u8 pad;
-};
-
-struct cpucp_hbm_row_replaced_rows_info {
- __le16 num_replaced_rows;
- __u8 pad[6];
- struct cpucp_hbm_row_info replaced_rows[CPUCP_HBM_ROW_REPLACE_MAX];
-};
-
-enum cpu_reset_status {
- CPU_RST_STATUS_NA = 0,
- CPU_RST_STATUS_SOFT_RST_DONE = 1,
-};
-
-#define SEC_PCR_DATA_BUF_SZ 256
-#define SEC_PCR_QUOTE_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */
-#define SEC_SIGNATURE_BUF_SZ 255 /* (256 - 1) 1 byte used for size */
-#define SEC_PUB_DATA_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */
-#define SEC_CERTIFICATE_BUF_SZ 2046 /* (2048 - 2) 2 bytes used for size */
-
-/*
- * struct cpucp_sec_attest_info - attestation report of the boot
- * @pcr_data: raw values of the PCR registers
- * @pcr_num_reg: number of PCR registers in the pcr_data array
- * @pcr_reg_len: length of each PCR register in the pcr_data array (bytes)
- * @nonce: number only used once. random number provided by host. this also
- * passed to the quote command as a qualifying data.
- * @pcr_quote_len: length of the attestation quote data (bytes)
- * @pcr_quote: attestation report data structure
- * @quote_sig_len: length of the attestation report signature (bytes)
- * @quote_sig: signature structure of the attestation report
- * @pub_data_len: length of the public data (bytes)
- * @public_data: public key for the signed attestation
- * (outPublic + name + qualifiedName)
- * @certificate_len: length of the certificate (bytes)
- * @certificate: certificate for the attestation signing key
- */
-struct cpucp_sec_attest_info {
- __u8 pcr_data[SEC_PCR_DATA_BUF_SZ];
- __u8 pcr_num_reg;
- __u8 pcr_reg_len;
- __le16 pad0;
- __le32 nonce;
- __le16 pcr_quote_len;
- __u8 pcr_quote[SEC_PCR_QUOTE_BUF_SZ];
- __u8 quote_sig_len;
- __u8 quote_sig[SEC_SIGNATURE_BUF_SZ];
- __le16 pub_data_len;
- __u8 public_data[SEC_PUB_DATA_BUF_SZ];
- __le16 certificate_len;
- __u8 certificate[SEC_CERTIFICATE_BUF_SZ];
-};
-
-/*
- * struct cpucp_dev_info_signed - device information signed by a secured device
- * @info: device information structure as defined above
- * @nonce: number only used once. random number provided by host. this number is
- * hashed and signed along with the device information.
- * @info_sig_len: length of the attestation signature (bytes)
- * @info_sig: signature of the info + nonce data.
- * @pub_data_len: length of the public data (bytes)
- * @public_data: public key info signed info data
- * (outPublic + name + qualifiedName)
- * @certificate_len: length of the certificate (bytes)
- * @certificate: certificate for the signing key
- */
-struct cpucp_dev_info_signed {
- struct cpucp_info info; /* assumed to be 64bit aligned */
- __le32 nonce;
- __le32 pad0;
- __u8 info_sig_len;
- __u8 info_sig[SEC_SIGNATURE_BUF_SZ];
- __le16 pub_data_len;
- __u8 public_data[SEC_PUB_DATA_BUF_SZ];
- __le16 certificate_len;
- __u8 certificate[SEC_CERTIFICATE_BUF_SZ];
-};
-
-#define DCORE_MON_REGS_SZ 512
-/*
- * struct dcore_monitor_regs_data - DCORE monitor regs data.
- * the structure follows sync manager block layout. Obsolete.
- * @mon_pay_addrl: array of payload address low bits.
- * @mon_pay_addrh: array of payload address high bits.
- * @mon_pay_data: array of payload data.
- * @mon_arm: array of monitor arm.
- * @mon_status: array of monitor status.
- */
-struct dcore_monitor_regs_data {
- __le32 mon_pay_addrl[DCORE_MON_REGS_SZ];
- __le32 mon_pay_addrh[DCORE_MON_REGS_SZ];
- __le32 mon_pay_data[DCORE_MON_REGS_SZ];
- __le32 mon_arm[DCORE_MON_REGS_SZ];
- __le32 mon_status[DCORE_MON_REGS_SZ];
-};
-
-/* contains SM data for each SYNC_MNGR (Obsolete) */
-struct cpucp_monitor_dump {
- struct dcore_monitor_regs_data sync_mngr_w_s;
- struct dcore_monitor_regs_data sync_mngr_e_s;
- struct dcore_monitor_regs_data sync_mngr_w_n;
- struct dcore_monitor_regs_data sync_mngr_e_n;
-};
-
-/*
- * The Type of the generic request (and other input arguments) will be fetched from user by reading
- * from "pkt_subidx" field in struct cpucp_packet.
- *
- * HL_PASSTHROUGHT_VERSIONS - Fetch all firmware versions.
- */
-enum hl_passthrough_type {
- HL_PASSTHROUGH_VERSIONS,
-};
-
-#endif /* CPUCP_IF_H */
diff --git a/drivers/accel/habanalabs/include/common/hl_boot_if.h b/drivers/accel/habanalabs/include/common/hl_boot_if.h
deleted file mode 100644
index cff79f7f9f75..000000000000
--- a/drivers/accel/habanalabs/include/common/hl_boot_if.h
+++ /dev/null
@@ -1,785 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
- * Copyright 2018-2020 HabanaLabs, Ltd.
- * All Rights Reserved.
- *
- */
-
-#ifndef HL_BOOT_IF_H
-#define HL_BOOT_IF_H
-
-#define LKD_HARD_RESET_MAGIC 0xED7BD694 /* deprecated - do not use */
-#define HL_POWER9_HOST_MAGIC 0x1DA30009
-
-#define BOOT_FIT_SRAM_OFFSET 0x200000
-
-#define VERSION_MAX_LEN 128
-
-enum cpu_boot_err {
- CPU_BOOT_ERR_DRAM_INIT_FAIL = 0,
- CPU_BOOT_ERR_FIT_CORRUPTED = 1,
- CPU_BOOT_ERR_TS_INIT_FAIL = 2,
- CPU_BOOT_ERR_DRAM_SKIPPED = 3,
- CPU_BOOT_ERR_BMC_WAIT_SKIPPED = 4,
- CPU_BOOT_ERR_NIC_DATA_NOT_RDY = 5,
- CPU_BOOT_ERR_NIC_FW_FAIL = 6,
- CPU_BOOT_ERR_SECURITY_NOT_RDY = 7,
- CPU_BOOT_ERR_SECURITY_FAIL = 8,
- CPU_BOOT_ERR_EFUSE_FAIL = 9,
- CPU_BOOT_ERR_PRI_IMG_VER_FAIL = 10,
- CPU_BOOT_ERR_SEC_IMG_VER_FAIL = 11,
- CPU_BOOT_ERR_PLL_FAIL = 12,
- CPU_BOOT_ERR_DEVICE_UNUSABLE_FAIL = 13,
- CPU_BOOT_ERR_BOOT_FW_CRIT_ERR = 18,
- CPU_BOOT_ERR_BINNING_FAIL = 19,
- CPU_BOOT_ERR_TPM_FAIL = 20,
- CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL = 21,
- CPU_BOOT_ERR_EEPROM_FAIL = 22,
- CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL = 23,
- CPU_BOOT_ERR_ENABLED = 31,
- CPU_BOOT_ERR_SCND_EN = 63,
- CPU_BOOT_ERR_LAST = 64 /* we have 2 registers of 32 bits */
-};
-
-/*
- * Mask for fatal failures
- * This mask contains all possible fatal failures, and a dynamic code
- * will clear the non-relevant ones.
- */
-#define CPU_BOOT_ERR_FATAL_MASK \
- ((1 << CPU_BOOT_ERR_DRAM_INIT_FAIL) | \
- (1 << CPU_BOOT_ERR_PLL_FAIL) | \
- (1 << CPU_BOOT_ERR_DEVICE_UNUSABLE_FAIL) | \
- (1 << CPU_BOOT_ERR_BINNING_FAIL) | \
- (1 << CPU_BOOT_ERR_DRAM_SKIPPED) | \
- (1 << CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL) | \
- (1 << CPU_BOOT_ERR_EEPROM_FAIL))
-
-/*
- * CPU error bits in BOOT_ERROR registers
- *
- * CPU_BOOT_ERR0_DRAM_INIT_FAIL DRAM initialization failed.
- * DRAM is not reliable to use.
- *
- * CPU_BOOT_ERR0_FIT_CORRUPTED FIT data integrity verification of the
- * image provided by the host has failed.
- *
- * CPU_BOOT_ERR0_TS_INIT_FAIL Thermal Sensor initialization failed.
- * Boot continues as usual, but keep in
- * mind this is a warning.
- *
- * CPU_BOOT_ERR0_DRAM_SKIPPED DRAM initialization has been skipped.
- * Skipping DRAM initialization has been
- * requested (e.g. strap, command, etc.)
- * and FW skipped the DRAM initialization.
- * Host can initialize the DRAM.
- *
- * CPU_BOOT_ERR0_BMC_WAIT_SKIPPED Waiting for BMC data will be skipped.
- * Meaning the BMC data might not be
- * available until reset.
- *
- * CPU_BOOT_ERR0_NIC_DATA_NOT_RDY NIC data from BMC is not ready.
- * BMC has not provided the NIC data yet.
- * Once provided this bit will be cleared.
- *
- * CPU_BOOT_ERR0_NIC_FW_FAIL NIC FW loading failed.
- * The NIC FW loading and initialization
- * failed. This means NICs are not usable.
- *
- * CPU_BOOT_ERR0_SECURITY_NOT_RDY Chip security initialization has been
- * started, but is not ready yet - chip
- * cannot be accessed.
- *
- * CPU_BOOT_ERR0_SECURITY_FAIL Security related tasks have failed.
- * The tasks are security init (root of
- * trust), boot authentication (chain of
- * trust), data packets authentication.
- *
- * CPU_BOOT_ERR0_EFUSE_FAIL Reading from eFuse failed.
- * The PCI device ID might be wrong.
- *
- * CPU_BOOT_ERR0_PRI_IMG_VER_FAIL Verification of primary image failed.
- * It mean that ppboot checksum
- * verification for the preboot primary
- * image has failed to match expected
- * checksum. Trying to program image again
- * might solve this.
- *
- * CPU_BOOT_ERR0_SEC_IMG_VER_FAIL Verification of secondary image failed.
- * It mean that ppboot checksum
- * verification for the preboot secondary
- * image has failed to match expected
- * checksum. Trying to program image again
- * might solve this.
- *
- * CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one
- * of the PLLs remains in REF_CLK
- *
- * CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL Device is unusable and customer support
- * should be contacted.
- *
- * CPU_BOOT_ERR0_BOOT_FW_CRIT_ERR Critical error was detected during
- * the execution of ppboot or preboot.
- * for example: stack overflow.
- *
- * CPU_BOOT_ERR0_BINNING_FAIL Binning settings failed, meaning
- * malfunctioning components might still be
- * in use.
- *
- * CPU_BOOT_ERR0_TPM_FAIL TPM verification flow failed.
- *
- * CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL Failed to set threshold for tmperature
- * sensor.
- *
- * CPU_BOOT_ERR_EEPROM_FAIL Failed reading EEPROM data. Defaults
- * are used.
- *
- * CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL Failed scrubbing the Engines/ARCFarm
- * memories. Boot disabled until reset.
- *
- * CPU_BOOT_ERR0_ENABLED Error registers enabled.
- * This is a main indication that the
- * running FW populates the error
- * registers. Meaning the error bits are
- * not garbage, but actual error statuses.
- */
-#define CPU_BOOT_ERR0_DRAM_INIT_FAIL (1 << CPU_BOOT_ERR_DRAM_INIT_FAIL)
-#define CPU_BOOT_ERR0_FIT_CORRUPTED (1 << CPU_BOOT_ERR_FIT_CORRUPTED)
-#define CPU_BOOT_ERR0_TS_INIT_FAIL (1 << CPU_BOOT_ERR_TS_INIT_FAIL)
-#define CPU_BOOT_ERR0_DRAM_SKIPPED (1 << CPU_BOOT_ERR_DRAM_SKIPPED)
-#define CPU_BOOT_ERR0_BMC_WAIT_SKIPPED (1 << CPU_BOOT_ERR_BMC_WAIT_SKIPPED)
-#define CPU_BOOT_ERR0_NIC_DATA_NOT_RDY (1 << CPU_BOOT_ERR_NIC_DATA_NOT_RDY)
-#define CPU_BOOT_ERR0_NIC_FW_FAIL (1 << CPU_BOOT_ERR_NIC_FW_FAIL)
-#define CPU_BOOT_ERR0_SECURITY_NOT_RDY (1 << CPU_BOOT_ERR_SECURITY_NOT_RDY)
-#define CPU_BOOT_ERR0_SECURITY_FAIL (1 << CPU_BOOT_ERR_SECURITY_FAIL)
-#define CPU_BOOT_ERR0_EFUSE_FAIL (1 << CPU_BOOT_ERR_EFUSE_FAIL)
-#define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL (1 << CPU_BOOT_ERR_PRI_IMG_VER_FAIL)
-#define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << CPU_BOOT_ERR_SEC_IMG_VER_FAIL)
-#define CPU_BOOT_ERR0_PLL_FAIL (1 << CPU_BOOT_ERR_PLL_FAIL)
-#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << CPU_BOOT_ERR_DEVICE_UNUSABLE_FAIL)
-#define CPU_BOOT_ERR0_BOOT_FW_CRIT_ERR (1 << CPU_BOOT_ERR_BOOT_FW_CRIT_ERR)
-#define CPU_BOOT_ERR0_BINNING_FAIL (1 << CPU_BOOT_ERR_BINNING_FAIL)
-#define CPU_BOOT_ERR0_TPM_FAIL (1 << CPU_BOOT_ERR_TPM_FAIL)
-#define CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL (1 << CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL)
-#define CPU_BOOT_ERR0_EEPROM_FAIL (1 << CPU_BOOT_ERR_EEPROM_FAIL)
-#define CPU_BOOT_ERR0_ENG_ARC_MEM_SCRUB_FAIL (1 << CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL)
-#define CPU_BOOT_ERR0_ENABLED (1 << CPU_BOOT_ERR_ENABLED)
-#define CPU_BOOT_ERR1_ENABLED (1 << CPU_BOOT_ERR_ENABLED)
-
-enum cpu_boot_dev_sts {
- CPU_BOOT_DEV_STS_SECURITY_EN = 0,
- CPU_BOOT_DEV_STS_DEBUG_EN = 1,
- CPU_BOOT_DEV_STS_WATCHDOG_EN = 2,
- CPU_BOOT_DEV_STS_DRAM_INIT_EN = 3,
- CPU_BOOT_DEV_STS_BMC_WAIT_EN = 4,
- CPU_BOOT_DEV_STS_E2E_CRED_EN = 5,
- CPU_BOOT_DEV_STS_HBM_CRED_EN = 6,
- CPU_BOOT_DEV_STS_RL_EN = 7,
- CPU_BOOT_DEV_STS_SRAM_SCR_EN = 8,
- CPU_BOOT_DEV_STS_DRAM_SCR_EN = 9,
- CPU_BOOT_DEV_STS_FW_HARD_RST_EN = 10,
- CPU_BOOT_DEV_STS_PLL_INFO_EN = 11,
- CPU_BOOT_DEV_STS_SP_SRAM_EN = 12,
- CPU_BOOT_DEV_STS_CLK_GATE_EN = 13,
- CPU_BOOT_DEV_STS_HBM_ECC_EN = 14,
- CPU_BOOT_DEV_STS_PKT_PI_ACK_EN = 15,
- CPU_BOOT_DEV_STS_FW_LD_COM_EN = 16,
- CPU_BOOT_DEV_STS_FW_IATU_CONF_EN = 17,
- CPU_BOOT_DEV_STS_FW_NIC_MAC_EN = 18,
- CPU_BOOT_DEV_STS_DYN_PLL_EN = 19,
- CPU_BOOT_DEV_STS_GIC_PRIVILEGED_EN = 20,
- CPU_BOOT_DEV_STS_EQ_INDEX_EN = 21,
- CPU_BOOT_DEV_STS_MULTI_IRQ_POLL_EN = 22,
- CPU_BOOT_DEV_STS_FW_NIC_STAT_XPCS91_EN = 23,
- CPU_BOOT_DEV_STS_FW_NIC_STAT_EXT_EN = 24,
- CPU_BOOT_DEV_STS_IS_IDLE_CHECK_EN = 25,
- CPU_BOOT_DEV_STS_MAP_HWMON_EN = 26,
- CPU_BOOT_DEV_STS_ENABLED = 31,
- CPU_BOOT_DEV_STS_SCND_EN = 63,
- CPU_BOOT_DEV_STS_LAST = 64 /* we have 2 registers of 32 bits */
-};
-
-/*
- * BOOT DEVICE STATUS bits in BOOT_DEVICE_STS registers
- *
- * CPU_BOOT_DEV_STS0_SECURITY_EN Security is Enabled.
- * This is an indication for security
- * enabled in FW, which means that
- * all conditions for security are met:
- * device is indicated as security enabled,
- * registers are protected, and device
- * uses keys for image verification.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_DEBUG_EN Debug is enabled.
- * Enabled when JTAG or DEBUG is enabled
- * in FW.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_WATCHDOG_EN Watchdog is enabled.
- * Watchdog is enabled in FW.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_DRAM_INIT_EN DRAM initialization is enabled.
- * DRAM initialization has been done in FW.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_BMC_WAIT_EN Waiting for BMC data enabled.
- * If set, it means that during boot,
- * FW waited for BMC data.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_E2E_CRED_EN E2E credits initialized.
- * FW initialized E2E credits.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_HBM_CRED_EN HBM credits initialized.
- * FW initialized HBM credits.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_RL_EN Rate limiter initialized.
- * FW initialized rate limiter.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_SRAM_SCR_EN SRAM scrambler enabled.
- * FW initialized SRAM scrambler.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_DRAM_SCR_EN DRAM scrambler enabled.
- * FW initialized DRAM scrambler.
- * Initialized in: u-boot
- *
- * CPU_BOOT_DEV_STS0_FW_HARD_RST_EN FW hard reset procedure is enabled.
- * FW has the hard reset procedure
- * implemented. This means that FW will
- * perform hard reset procedure on
- * receiving the halt-machine event.
- * Initialized in: preboot, u-boot, linux
- *
- * CPU_BOOT_DEV_STS0_PLL_INFO_EN FW retrieval of PLL info is enabled.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_SP_SRAM_EN SP SRAM is initialized and available
- * for use.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_CLK_GATE_EN Clock Gating enabled.
- * FW initialized Clock Gating.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_HBM_ECC_EN HBM ECC handling Enabled.
- * FW handles HBM ECC indications.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN Packets ack value used in the armcpd
- * is set to the PI counter.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_FW_LD_COM_EN Flexible FW loading communication
- * protocol is enabled.
- * Initialized in: preboot
- *
- * CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN FW iATU configuration is enabled.
- * This bit if set, means the iATU has been
- * configured and is ready for use.
- * Initialized in: ppboot
- *
- * CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN NIC MAC channels init is done by FW and
- * any access to them is done via the FW.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_DYN_PLL_EN Dynamic PLL configuration is enabled.
- * FW sends to host a bitmap of supported
- * PLLs.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN GIC access permission only from
- * previleged entity. FW sets this status
- * bit for host. If this bit is set then
- * GIC can not be accessed from host.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_EQ_INDEX_EN Event Queue (EQ) index is a running
- * index for each new event sent to host.
- * This is used as a method in host to
- * identify that the waiting event in
- * queue is actually a new event which
- * was not served before.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN Use multiple scratchpad interfaces to
- * prevent IRQs overriding each other.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN
- * NIC STAT and XPCS91 access is restricted
- * and is done via FW only.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN
- * NIC STAT get all is supported.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN
- * F/W checks if the device is idle by reading defined set
- * of registers. It returns a bitmask of all the engines,
- * where a bit is set if the engine is not idle.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_MAP_HWMON_EN
- * If set, means f/w supports proprietary
- * HWMON enum mapping to cpucp enums.
- * Initialized in: linux
- *
- * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.
- * This is a main indication that the
- * running FW populates the device status
- * register. Meaning the device status
- * bits are not garbage, but actual
- * statuses.
- * Initialized in: preboot
- *
- */
-#define CPU_BOOT_DEV_STS0_SECURITY_EN (1 << CPU_BOOT_DEV_STS_SECURITY_EN)
-#define CPU_BOOT_DEV_STS0_DEBUG_EN (1 << CPU_BOOT_DEV_STS_DEBUG_EN)
-#define CPU_BOOT_DEV_STS0_WATCHDOG_EN (1 << CPU_BOOT_DEV_STS_WATCHDOG_EN)
-#define CPU_BOOT_DEV_STS0_DRAM_INIT_EN (1 << CPU_BOOT_DEV_STS_DRAM_INIT_EN)
-#define CPU_BOOT_DEV_STS0_BMC_WAIT_EN (1 << CPU_BOOT_DEV_STS_BMC_WAIT_EN)
-#define CPU_BOOT_DEV_STS0_E2E_CRED_EN (1 << CPU_BOOT_DEV_STS_E2E_CRED_EN)
-#define CPU_BOOT_DEV_STS0_HBM_CRED_EN (1 << CPU_BOOT_DEV_STS_HBM_CRED_EN)
-#define CPU_BOOT_DEV_STS0_RL_EN (1 << CPU_BOOT_DEV_STS_RL_EN)
-#define CPU_BOOT_DEV_STS0_SRAM_SCR_EN (1 << CPU_BOOT_DEV_STS_SRAM_SCR_EN)
-#define CPU_BOOT_DEV_STS0_DRAM_SCR_EN (1 << CPU_BOOT_DEV_STS_DRAM_SCR_EN)
-#define CPU_BOOT_DEV_STS0_FW_HARD_RST_EN (1 << CPU_BOOT_DEV_STS_FW_HARD_RST_EN)
-#define CPU_BOOT_DEV_STS0_PLL_INFO_EN (1 << CPU_BOOT_DEV_STS_PLL_INFO_EN)
-#define CPU_BOOT_DEV_STS0_SP_SRAM_EN (1 << CPU_BOOT_DEV_STS_SP_SRAM_EN)
-#define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << CPU_BOOT_DEV_STS_CLK_GATE_EN)
-#define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << CPU_BOOT_DEV_STS_HBM_ECC_EN)
-#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << CPU_BOOT_DEV_STS_PKT_PI_ACK_EN)
-#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << CPU_BOOT_DEV_STS_FW_LD_COM_EN)
-#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << CPU_BOOT_DEV_STS_FW_IATU_CONF_EN)
-#define CPU_BOOT_DEV_STS0_FW_NIC_MAC_EN (1 << CPU_BOOT_DEV_STS_FW_NIC_MAC_EN)
-#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << CPU_BOOT_DEV_STS_DYN_PLL_EN)
-#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << CPU_BOOT_DEV_STS_GIC_PRIVILEGED_EN)
-#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << CPU_BOOT_DEV_STS_EQ_INDEX_EN)
-#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << CPU_BOOT_DEV_STS_MULTI_IRQ_POLL_EN)
-#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_XPCS91_EN (1 << CPU_BOOT_DEV_STS_FW_NIC_STAT_XPCS91_EN)
-#define CPU_BOOT_DEV_STS0_FW_NIC_STAT_EXT_EN (1 << CPU_BOOT_DEV_STS_FW_NIC_STAT_EXT_EN)
-#define CPU_BOOT_DEV_STS0_IS_IDLE_CHECK_EN (1 << CPU_BOOT_DEV_STS_IS_IDLE_CHECK_EN)
-#define CPU_BOOT_DEV_STS0_MAP_HWMON_EN (1 << CPU_BOOT_DEV_STS_MAP_HWMON_EN)
-#define CPU_BOOT_DEV_STS0_ENABLED (1 << CPU_BOOT_DEV_STS_ENABLED)
-#define CPU_BOOT_DEV_STS1_ENABLED (1 << CPU_BOOT_DEV_STS_ENABLED)
-
-enum cpu_boot_status {
- CPU_BOOT_STATUS_NA = 0, /* Default value after reset of chip */
- CPU_BOOT_STATUS_IN_WFE = 1,
- CPU_BOOT_STATUS_DRAM_RDY = 2,
- CPU_BOOT_STATUS_SRAM_AVAIL = 3,
- CPU_BOOT_STATUS_IN_BTL = 4, /* BTL is H/W FSM */
- CPU_BOOT_STATUS_IN_PREBOOT = 5,
- CPU_BOOT_STATUS_IN_SPL, /* deprecated - not reported */
- CPU_BOOT_STATUS_IN_UBOOT = 7,
- CPU_BOOT_STATUS_DRAM_INIT_FAIL, /* deprecated - will be removed */
- CPU_BOOT_STATUS_FIT_CORRUPTED, /* deprecated - will be removed */
- /* U-Boot console prompt activated, commands are not processed */
- CPU_BOOT_STATUS_UBOOT_NOT_READY = 10,
- /* Finished NICs init, reported after DRAM and NICs */
- CPU_BOOT_STATUS_NIC_FW_RDY = 11,
- CPU_BOOT_STATUS_TS_INIT_FAIL, /* deprecated - will be removed */
- CPU_BOOT_STATUS_DRAM_SKIPPED, /* deprecated - will be removed */
- CPU_BOOT_STATUS_BMC_WAITING_SKIPPED, /* deprecated - will be removed */
- /* Last boot loader progress status, ready to receive commands */
- CPU_BOOT_STATUS_READY_TO_BOOT = 15,
- /* Internal Boot finished, ready for boot-fit */
- CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT = 16,
- /* Internal Security has been initialized, device can be accessed */
- CPU_BOOT_STATUS_SECURITY_READY = 17,
-};
-
-enum kmd_msg {
- KMD_MSG_NA = 0,
- KMD_MSG_GOTO_WFE,
- KMD_MSG_FIT_RDY,
- KMD_MSG_SKIP_BMC,
- RESERVED,
- KMD_MSG_RST_DEV,
- KMD_MSG_LAST
-};
-
-enum cpu_msg_status {
- CPU_MSG_CLR = 0,
- CPU_MSG_OK,
- CPU_MSG_ERR,
-};
-
-/* communication registers mapping - consider ABI when changing */
-struct cpu_dyn_regs {
- __le32 cpu_pq_base_addr_low;
- __le32 cpu_pq_base_addr_high;
- __le32 cpu_pq_length;
- __le32 cpu_pq_init_status;
- __le32 cpu_eq_base_addr_low;
- __le32 cpu_eq_base_addr_high;
- __le32 cpu_eq_length;
- __le32 cpu_eq_ci;
- __le32 cpu_cq_base_addr_low;
- __le32 cpu_cq_base_addr_high;
- __le32 cpu_cq_length;
- __le32 cpu_pf_pq_pi;
- __le32 cpu_boot_dev_sts0;
- __le32 cpu_boot_dev_sts1;
- __le32 cpu_boot_err0;
- __le32 cpu_boot_err1;
- __le32 cpu_boot_status;
- __le32 fw_upd_sts;
- __le32 fw_upd_cmd;
- __le32 fw_upd_pending_sts;
- __le32 fuse_ver_offset;
- __le32 preboot_ver_offset;
- __le32 uboot_ver_offset;
- __le32 hw_state;
- __le32 kmd_msg_to_cpu;
- __le32 cpu_cmd_status_to_host;
- __le32 gic_host_pi_upd_irq;
- __le32 gic_tpc_qm_irq_ctrl;
- __le32 gic_mme_qm_irq_ctrl;
- __le32 gic_dma_qm_irq_ctrl;
- __le32 gic_nic_qm_irq_ctrl;
- __le32 gic_dma_core_irq_ctrl;
- __le32 gic_host_halt_irq;
- __le32 gic_host_ints_irq;
- __le32 gic_host_soft_rst_irq;
- __le32 gic_rot_qm_irq_ctrl;
- __le32 cpu_rst_status;
- __le32 eng_arc_irq_ctrl;
- __le32 reserved1[20]; /* reserve for future use */
-};
-
-/* TODO: remove the desc magic after the code is updated to use message */
-/* HCDM - Habana Communications Descriptor Magic */
-#define HL_COMMS_DESC_MAGIC 0x4843444D
-#define HL_COMMS_DESC_VER 3
-
-/* HCMv - Habana Communications Message + header version */
-#define HL_COMMS_MSG_MAGIC_VALUE 0x48434D00
-#define HL_COMMS_MSG_MAGIC_MASK 0xFFFFFF00
-#define HL_COMMS_MSG_MAGIC_VER_MASK 0xFF
-
-#define HL_COMMS_MSG_MAGIC_VER(ver) (HL_COMMS_MSG_MAGIC_VALUE | \
- ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
-#define HL_COMMS_MSG_MAGIC_V0 HL_COMMS_DESC_MAGIC
-#define HL_COMMS_MSG_MAGIC_V1 HL_COMMS_MSG_MAGIC_VER(1)
-#define HL_COMMS_MSG_MAGIC_V2 HL_COMMS_MSG_MAGIC_VER(2)
-#define HL_COMMS_MSG_MAGIC_V3 HL_COMMS_MSG_MAGIC_VER(3)
-
-#define HL_COMMS_MSG_MAGIC HL_COMMS_MSG_MAGIC_V3
-
-#define HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC(magic) \
- (((magic) & HL_COMMS_MSG_MAGIC_MASK) == \
- HL_COMMS_MSG_MAGIC_VALUE)
-
-#define HL_COMMS_MSG_MAGIC_VALIDATE_VERSION(magic, ver) \
- (((magic) & HL_COMMS_MSG_MAGIC_VER_MASK) >= \
- ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
-
-#define HL_COMMS_MSG_MAGIC_VALIDATE(magic, ver) \
- (HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC((magic)) && \
- HL_COMMS_MSG_MAGIC_VALIDATE_VERSION((magic), (ver)))
-
-enum comms_msg_type {
- HL_COMMS_DESC_TYPE = 0,
- HL_COMMS_RESET_CAUSE_TYPE = 1,
- HL_COMMS_FW_CFG_SKIP_TYPE = 2,
- HL_COMMS_BINNING_CONF_TYPE = 3,
-};
-
-/*
- * Binning information shared between LKD and FW
- * @tpc_mask_l - TPC binning information lower 64 bit
- * @dec_mask - Decoder binning information
- * @dram_mask - DRAM binning information
- * @edma_mask - EDMA binning information
- * @mme_mask_l - MME binning information lower 32
- * @mme_mask_h - MME binning information upper 32
- * @rot_mask - Rotator binning information
- * @xbar_mask - xBAR binning information
- * @reserved - reserved field for future binning info w/o ABI change
- * @tpc_mask_h - TPC binning information upper 64 bit
- * @nic_mask - NIC binning information
- */
-struct lkd_fw_binning_info {
- __le64 tpc_mask_l;
- __le32 dec_mask;
- __le32 dram_mask;
- __le32 edma_mask;
- __le32 mme_mask_l;
- __le32 mme_mask_h;
- __le32 rot_mask;
- __le32 xbar_mask;
- __le32 reserved0;
- __le64 tpc_mask_h;
- __le64 nic_mask;
- __le32 reserved1[8];
-};
-
-/* TODO: remove this struct after the code is updated to use message */
-/* this is the comms descriptor header - meta data */
-struct comms_desc_header {
- __le32 magic; /* magic for validation */
- __le32 crc32; /* CRC32 of the descriptor w/o header */
- __le16 size; /* size of the descriptor w/o header */
- __u8 version; /* descriptor version */
- __u8 reserved[5]; /* pad to 64 bit */
-};
-
-/* this is the comms message header - meta data */
-struct comms_msg_header {
- __le32 magic; /* magic for validation */
- __le32 crc32; /* CRC32 of the message w/o header */
- __le16 size; /* size of the message w/o header */
- __u8 version; /* message payload version */
- __u8 type; /* message type */
- __u8 reserved[4]; /* pad to 64 bit */
-};
-
-enum lkd_fw_ascii_msg_lvls {
- LKD_FW_ASCII_MSG_ERR = 0,
- LKD_FW_ASCII_MSG_WRN = 1,
- LKD_FW_ASCII_MSG_INF = 2,
- LKD_FW_ASCII_MSG_DBG = 3,
-};
-
-#define LKD_FW_ASCII_MSG_MAX_LEN 128
-#define LKD_FW_ASCII_MSG_MAX 4 /* consider ABI when changing */
-
-struct lkd_fw_ascii_msg {
- __u8 valid;
- __u8 msg_lvl;
- __u8 reserved[6];
- char msg[LKD_FW_ASCII_MSG_MAX_LEN];
-};
-
-/* this is the main FW descriptor - consider ABI when changing */
-struct lkd_fw_comms_desc {
- struct comms_desc_header header;
- struct cpu_dyn_regs cpu_dyn_regs;
- char fuse_ver[VERSION_MAX_LEN];
- char cur_fw_ver[VERSION_MAX_LEN];
- /* can be used for 1 more version w/o ABI change */
- char reserved0[VERSION_MAX_LEN];
- __le64 img_addr; /* address for next FW component load */
- struct lkd_fw_binning_info binning_info;
- struct lkd_fw_ascii_msg ascii_msg[LKD_FW_ASCII_MSG_MAX];
-};
-
-enum comms_reset_cause {
- HL_RESET_CAUSE_UNKNOWN = 0,
- HL_RESET_CAUSE_HEARTBEAT = 1,
- HL_RESET_CAUSE_TDR = 2,
-};
-
-/* TODO: remove define after struct name is aligned on all projects */
-#define lkd_msg_comms lkd_fw_comms_msg
-
-/* this is the comms message descriptor */
-struct lkd_fw_comms_msg {
- struct comms_msg_header header;
- /* union for future expantions of new messages */
- union {
- struct {
- struct cpu_dyn_regs cpu_dyn_regs;
- char fuse_ver[VERSION_MAX_LEN];
- char cur_fw_ver[VERSION_MAX_LEN];
- /* can be used for 1 more version w/o ABI change */
- char reserved0[VERSION_MAX_LEN];
- /* address for next FW component load */
- __le64 img_addr;
- struct lkd_fw_binning_info binning_info;
- struct lkd_fw_ascii_msg ascii_msg[LKD_FW_ASCII_MSG_MAX];
- };
- struct {
- __u8 reset_cause;
- };
- struct {
- __u8 fw_cfg_skip; /* 1 - skip, 0 - don't skip */
- };
- struct lkd_fw_binning_info binning_conf;
- };
-};
-
-/*
- * LKD commands:
- *
- * COMMS_NOOP Used to clear the command register and no actual
- * command is send.
- *
- * COMMS_CLR_STS Clear status command - FW should clear the
- * status register. Used for synchronization
- * between the commands as part of the race free
- * protocol.
- *
- * COMMS_RST_STATE Reset the current communication state which is
- * kept by FW for proper responses.
- * Should be used in the beginning of the
- * communication cycle to clean any leftovers from
- * previous communication attempts.
- *
- * COMMS_PREP_DESC Prepare descriptor for setting up the
- * communication and other dynamic data:
- * struct lkd_fw_comms_desc.
- * This command has a parameter stating the next FW
- * component size, so the FW can actually prepare a
- * space for it and in the status response provide
- * the descriptor offset. The Offset of the next FW
- * data component is a part of the descriptor
- * structure.
- *
- * COMMS_DATA_RDY The FW data has been uploaded and is ready for
- * validation.
- *
- * COMMS_EXEC Execute the next FW component.
- *
- * COMMS_RST_DEV Reset the device.
- *
- * COMMS_GOTO_WFE Execute WFE command. Allowed only on non-secure
- * devices.
- *
- * COMMS_SKIP_BMC Perform actions required for BMC-less servers.
- * Do not wait for BMC response.
- *
- * COMMS_PREP_DESC_ELBI Same as COMMS_PREP_DESC only that the memory
- * space is allocated in a ELBI access only
- * address range.
- *
- */
-enum comms_cmd {
- COMMS_NOOP = 0,
- COMMS_CLR_STS = 1,
- COMMS_RST_STATE = 2,
- COMMS_PREP_DESC = 3,
- COMMS_DATA_RDY = 4,
- COMMS_EXEC = 5,
- COMMS_RST_DEV = 6,
- COMMS_GOTO_WFE = 7,
- COMMS_SKIP_BMC = 8,
- COMMS_PREP_DESC_ELBI = 10,
- COMMS_INVLD_LAST
-};
-
-#define COMMS_COMMAND_SIZE_SHIFT 0
-#define COMMS_COMMAND_SIZE_MASK 0x1FFFFFF
-#define COMMS_COMMAND_CMD_SHIFT 27
-#define COMMS_COMMAND_CMD_MASK 0xF8000000
-
-/*
- * LKD command to FW register structure
- * @size - FW component size
- * @cmd - command from enum comms_cmd
- */
-struct comms_command {
- union { /* bit fields are only for FW use */
- struct {
- u32 size :25; /* 32MB max. */
- u32 reserved :2;
- enum comms_cmd cmd :5; /* 32 commands */
- };
- __le32 val;
- };
-};
-
-/*
- * FW status
- *
- * COMMS_STS_NOOP Used to clear the status register and no actual
- * status is provided.
- *
- * COMMS_STS_ACK Command has been received and recognized.
- *
- * COMMS_STS_OK Command execution has finished successfully.
- *
- * COMMS_STS_ERR Command execution was unsuccessful and resulted
- * in error.
- *
- * COMMS_STS_VALID_ERR FW validation has failed.
- *
- * COMMS_STS_TIMEOUT_ERR Command execution has timed out.
- */
-enum comms_sts {
- COMMS_STS_NOOP = 0,
- COMMS_STS_ACK = 1,
- COMMS_STS_OK = 2,
- COMMS_STS_ERR = 3,
- COMMS_STS_VALID_ERR = 4,
- COMMS_STS_TIMEOUT_ERR = 5,
- COMMS_STS_INVLD_LAST
-};
-
-/* RAM types for FW components loading - defines the base address */
-enum comms_ram_types {
- COMMS_SRAM = 0,
- COMMS_DRAM = 1,
-};
-
-#define COMMS_STATUS_OFFSET_SHIFT 0
-#define COMMS_STATUS_OFFSET_MASK 0x03FFFFFF
-#define COMMS_STATUS_OFFSET_ALIGN_SHIFT 2
-#define COMMS_STATUS_RAM_TYPE_SHIFT 26
-#define COMMS_STATUS_RAM_TYPE_MASK 0x0C000000
-#define COMMS_STATUS_STATUS_SHIFT 28
-#define COMMS_STATUS_STATUS_MASK 0xF0000000
-
-/*
- * FW status to LKD register structure
- * @offset - an offset from the base of the ram_type shifted right by
- * 2 bits (always aligned to 32 bits).
- * Allows a maximum addressable offset of 256MB from RAM base.
- * Example: for real offset in RAM of 0x800000 (8MB), the value
- * in offset field is (0x800000 >> 2) = 0x200000.
- * @ram_type - the RAM type that should be used for offset from
- * enum comms_ram_types
- * @status - status from enum comms_sts
- */
-struct comms_status {
- union { /* bit fields are only for FW use */
- struct {
- u32 offset :26;
- enum comms_ram_types ram_type :2;
- enum comms_sts status :4; /* 16 statuses */
- };
- __le32 val;
- };
-};
-
-#define NAME_MAX_LEN 32 /* bytes */
-struct hl_module_data {
- __u8 name[NAME_MAX_LEN];
- __u8 version[VERSION_MAX_LEN];
-};
-
-/**
- * struct hl_component_versions - versions associated with hl component.
- * @struct_size: size of all the struct (including dynamic size of modules).
- * @modules_offset: offset of the modules field in this struct.
- * @component: version of the component itself.
- * @fw_os: Firmware OS Version.
- * @comp_name: Name of the component.
- * @modules_counter: number of set bits in modules_mask.
- * @reserved: reserved for future use.
- * @modules: versions of the component's modules. Elborated explanation in
- * struct cpucp_versions.
- */
-struct hl_component_versions {
- __le16 struct_size;
- __le16 modules_offset;
- __u8 component[VERSION_MAX_LEN];
- __u8 fw_os[VERSION_MAX_LEN];
- __u8 comp_name[NAME_MAX_LEN];
- __u8 modules_counter;
- __u8 reserved[3];
- struct hl_module_data modules[];
-};
-
-/* Max size of fit size */
-#define HL_FW_VERSIONS_FIT_SIZE 4096
-
-#endif /* HL_BOOT_IF_H */
diff --git a/drivers/accel/habanalabs/include/gaudi/gaudi_fw_if.h b/drivers/accel/habanalabs/include/gaudi/gaudi_fw_if.h
index 2dba02757d37..a2547f306750 100644
--- a/drivers/accel/habanalabs/include/gaudi/gaudi_fw_if.h
+++ b/drivers/accel/habanalabs/include/gaudi/gaudi_fw_if.h
@@ -44,38 +44,6 @@ struct eq_nic_sei_event {
__u8 pad[6];
};
-/*
- * struct gaudi_nic_status - describes the status of a NIC port.
- * @port: NIC port index.
- * @bad_format_cnt: e.g. CRC.
- * @responder_out_of_sequence_psn_cnt: e.g NAK.
- * @high_ber_reinit_cnt: link reinit due to high BER.
- * @correctable_err_cnt: e.g. bit-flip.
- * @uncorrectable_err_cnt: e.g. MAC errors.
- * @retraining_cnt: re-training counter.
- * @up: is port up.
- * @pcs_link: has PCS link.
- * @phy_ready: is PHY ready.
- * @auto_neg: is Autoneg enabled.
- * @timeout_retransmission_cnt: timeout retransmission events
- * @high_ber_cnt: high ber events
- */
-struct gaudi_nic_status {
- __u32 port;
- __u32 bad_format_cnt;
- __u32 responder_out_of_sequence_psn_cnt;
- __u32 high_ber_reinit;
- __u32 correctable_err_cnt;
- __u32 uncorrectable_err_cnt;
- __u32 retraining_cnt;
- __u8 up;
- __u8 pcs_link;
- __u8 phy_ready;
- __u8 auto_neg;
- __u32 timeout_retransmission_cnt;
- __u32 high_ber_cnt;
-};
-
struct gaudi_cold_rst_data {
union {
struct {
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
index f661068d0c5f..a426410139af 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_events.h
@@ -959,6 +959,13 @@ enum gaudi2_async_event_id {
GAUDI2_EVENT_ARC_DCCM_FULL = 1319,
GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED = 1320,
GAUDI2_EVENT_CPU_DEV_RESET_REQ = 1321,
+ GAUDI2_EVENT_ARC_PWR_BRK_ENTRY = 1322,
+ GAUDI2_EVENT_ARC_PWR_BRK_EXT = 1323,
+ GAUDI2_EVENT_ARC_PWR_RD_MODE0 = 1324,
+ GAUDI2_EVENT_ARC_PWR_RD_MODE1 = 1325,
+ GAUDI2_EVENT_ARC_PWR_RD_MODE2 = 1326,
+ GAUDI2_EVENT_ARC_PWR_RD_MODE3 = 1327,
+ GAUDI2_EVENT_ARC_EQ_HEARTBEAT = 1328,
GAUDI2_EVENT_SIZE,
};
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
index ad01fc4e9940..b2dbe1f64430 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
@@ -1293,7 +1293,7 @@ static struct gaudi2_async_events_ids_map gaudi2_irq_map_table[] = {
.name = "" },
{ .fc_id = 631, .cpu_id = 128, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
.name = "PCIE_P2P_MSIX" },
- { .fc_id = 632, .cpu_id = 129, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
+ { .fc_id = 632, .cpu_id = 129, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_HARD,
.name = "PCIE_DRAIN_COMPLETE" },
{ .fc_id = 633, .cpu_id = 130, .valid = 1, .msg = 0, .reset = EVENT_RESET_TYPE_NONE,
.name = "TPC0_BMON_SPMU" },
@@ -2673,6 +2673,20 @@ static struct gaudi2_async_events_ids_map gaudi2_irq_map_table[] = {
.name = "FP32_NOT_SUPPORTED" },
{ .fc_id = 1321, .cpu_id = 627, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
.name = "DEV_RESET_REQ" },
+ { .fc_id = 1322, .cpu_id = 628, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_BRK_ENTRY" },
+ { .fc_id = 1323, .cpu_id = 629, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_BRK_EXT" },
+ { .fc_id = 1324, .cpu_id = 630, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_RD_MODE0" },
+ { .fc_id = 1325, .cpu_id = 631, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_RD_MODE1" },
+ { .fc_id = 1326, .cpu_id = 632, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_RD_MODE2" },
+ { .fc_id = 1327, .cpu_id = 633, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_PWR_RD_MODE3" },
+ { .fc_id = 1328, .cpu_id = 634, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_EQ_HEARTBEAT" },
};
#endif /* __GAUDI2_ASYNC_IDS_MAP_EVENTS_EXT_H_ */