Linux kernel tracing

Intel® System Debugger User Guide

ID	Date	Version	Classification
648476	10/17/2024		Confidential

Linux kernel tracing allows to pipe dmesg, userspace messages and kernel ftrace events through Intel(R) TraceHub resulting in a common firmware/kernel log.

07_Linux_Tracing_Flow

The Linux kernel patches which are required for ftrace event tracing are shipped with the trace tool since they are not part of the upstream Linux kernel yet.

Patchset is based on Linux v6.1-rc1 and is tested against Ubuntu 22.04 LTS 5.15.0-41 kernel

git format-patch -v6 -o outgoing –base=v6.1-rc1 -3

The linux kernel patches can be found in Intel(R) System Trace installation folder, i.e.:

C:/IntelSWTools/system_debugger/XXXX-nda/system_trace/agent/plugins/linux_trace/kernel_patch/linux_ftrace_support.zip

SHA256 9cceef2ec6af237a6eaa3391615f67134edbd2c60ad42cbbf2299c9d6adcbbe0

MD5 0fb47da9a86becfd35560eebf81ed5c0

Prerequisites

Following modules must be built as part of kernel configuration and loaded during startup or manually before any of described further steps are executed:
- Intel(R) TraceHub modules
  intel_th_msu
  intel_th_sth
- STM Framework modules
  stm_console
  stm_ftrace
  stm_p_sys-t
Target agent must be copied/installed on target machine
BIOS setup
- Enable debug consent
- Leave other switches in Advanced debug settings default
Kernel commandline setup
To allow kernel write into Intel(R) TraceHub you need to switch intel_th driver into Host mode
1. Append Linux command line
2. Update bootloader settings sudo update-grub or sudo update-grub2
3. Reboot the machine
4. Check the setting was applied

Setup STM policy

This part is automatically set up by OnTarget agent.

Below setup script are provided for undertstanding and reference what agent does:

#!/bin/bash
set -x
set -e

# mount configfs
mkdir -p /config
mount -t configfs none /config

# create STP policy
# default policy - if not created and any other can be applied - writer gets -EINVAL
# NPK masters 257-258
mkdir -p /config/stp-policy/0-sth:p_sys-t.my-policy/default/
echo 257 258 > /config/stp-policy/0-sth:p_sys-t.my-policy/default/masters

# printk policy (1)
# NPK masters 261-262
mkdir -p /config/stp-policy/0-sth:p_sys-t.my-policy/console/
echo 261 262 > /config/stp-policy/0-sth:p_sys-t.my-policy/console/masters
echo 0-sth > /sys/class/stm_source/console/stm_source_link

# ftrace policy (2)
# NPK masters 259-260
echo 259 260 > /config/stp-policy/0-sth:p_sys-t.my-policy/ftrace/masters
# UUID should correspond to trace extension catalog
echo 6e5dd011-0a15-4380-b413-d1e61ce9d005 > /config/stp-policy/0-sth:p_sys-t.my-policy/ftrace/uuid
echo 0-sth > /sys/class/stm_source/ftrace/stm_source_link

Start tracing

TraceCLI

Start OnTarget trace agent: sudo ./agent/bin/intel_trace_agent with root permissions on SUT
Start TraceCLI and connect to target agent trace_agent.connect(ip="<ip-address>", port=1534)
Start regular trace capture trace.start_capture()
After test run (i.e. problem reproducer) stop the capture with trace.stop_capture()
Decode the file with trace.decode() command or use Eclipse UI to decode the trace capture

Eclipse UI

Linux OnTarget tracing flow is not yet supported in Eclipse UI

Known limitations

Passing arbitrary strings through ftrace events is not supported.

Example driver

Define 2 events in test driver: one for loading/unloading (th_test_trace_mod) and one timer event (th_test_trace_timer)

#undef TRACE_SYSTEM
#define TRACE_SYSTEM th_test_trace

#if !defined(_TRACE_TH_TEST_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_TH_TEST_TRACE_H

#include <linux/tracepoint.h>

TRACE_EVENT(th_test_trace_mod,
    TP_PROTO(bool start),
    TP_ARGS(start),
    TP_STRUCT__entry(
        __field(bool,   start)
    ),
    TP_fast_assign(
        __entry->start = start;
    ),
    TP_printk("Module: %s", __entry->start ? "start" : "end")
);

TRACE_EVENT(th_test_trace_timer,
    TP_PROTO(uint32_t counter, const char* text),
    TP_ARGS(counter, text),
    TP_STRUCT__entry(
        __field(uint32_t,   counter)
        __field(char,   text[14])
    ),
    TP_fast_assign(
        __entry->counter = counter;
        memcpy(__entry->text, text, strlen(text));
    ),
    TP_printk("event: TraceHub test counter: %u %s", __entry->counter, __entry->text)
);

#endif

#include <trace/define_trace.h>

Use these events in your kernel driver i.e. trace_th_test_trace_timer(…)

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/timer.h>

#define CREATE_TRACE_POINTS
#include <trace/events/th_test_trace.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Mikhail Lappo");
MODULE_DESCRIPTION("A test kernel module for Intel TraceHub.");
MODULE_VERSION("0.01");

static struct timer_list periodic_event_timer;
static uint32_t timer_counter = 0;

void fire_test_event(struct timer_list* timer)
{
    const char t1[14] = "*** test1 ***";
    const char t2[14] = "*** test2 ***";
    const char t3[14] = "*** test3 ***";
    printk(KERN_ERR "-----------------------------------------------\n");
    printk(KERN_ERR "printk: Timer event in TH %u\n", timer_counter);
    trace_th_test_trace_timer(timer_counter++, t1);
    printk(KERN_ERR "-----------------------------------------------\n");
    mod_timer(&periodic_event_timer, jiffies + msecs_to_jiffies(1000));
}

static int __init intel_th_test_init(void) {
    printk(KERN_INFO "Hello, Intel TH!\n");
    timer_setup(&periodic_event_timer, fire_test_event, 0);
    mod_timer(&periodic_event_timer, jiffies + msecs_to_jiffies(1000));
    trace_th_test_trace_mod(true);
    return 0;
}
static void __exit intel_th_test_exit(void) {
    trace_th_test_trace_mod(false);
    del_timer(&periodic_event_timer);
    printk(KERN_INFO "Goodbye, Intel TH!\n");
}

module_init(intel_th_test_init);
module_exit(intel_th_test_exit);