Skip to main content

Bare-Metal Servers - Solana - PoH Binding

Allnodes Team avatar
Written by Allnodes Team
Updated over 3 weeks ago

How to bind 2nd core for PoH tasks

  1. Run the following script as root:

    cat << \EOF | bash
    #!/bin/bash

    # Core ID to isolate
    PHYSICAL_CORE_ID=2

    function run_lscpu {
    lscpu -J | jq -rc '.lscpu | map( { (.field): .data } ) | add'
    }

    function main {
    cpu_data=$(run_lscpu)
    if [[ -z $cpu_data ]]; then
    echo "Unable to retrieve CPU data, exiting..." >&2
    exit 1
    fi

    local num_sockets=$(jq -rc '."Socket(s):"' <<< "${cpu_data}" 2>/dev/null)
    local cores_per_socket=$(jq -rc '."Core(s) per socket:"' <<< "${cpu_data}" 2>/dev/null)

    num_cores=$((num_sockets * cores_per_socket))
    echo "Physical cores: ${num_cores}"

    threads_per_core=$(jq -rc '."Thread(s) per core:"' <<< "${cpu_data}" 2>/dev/null)
    echo "Threads per core: ${threads_per_core}"

    if [[ $PHYSICAL_CORE_ID -ge $num_cores ]]; then
    echo "Invalid Core ID specified, allowed values: 0-$((num_cores-1))" >&2
    exit 1
    fi

    local thread_ids=()
    for ((i=0; i<$threads_per_core; i++)); do
    thread_ids+=("$((PHYSICAL_CORE_ID+i*cores_per_socket))")
    done

    local start_range=0
    local thread_ids_inverted=()
    for ((i=0; i<=$threads_per_core; i++)); do
    local end_range
    if [[ $i -eq $threads_per_core ]]; then
    end_range=$((num_cores*threads_per_core-1))
    else
    end_range="$((PHYSICAL_CORE_ID+i*cores_per_socket-1))"
    fi
    thread_ids_inverted+=("${start_range}-${end_range}")
    start_range="$((PHYSICAL_CORE_ID+i*cores_per_socket+1))"
    done

    local thread_ids_str=$(IFS=,; echo "${thread_ids[*]}")
    local thread_ids_inverted_str=$(IFS=,; echo "${thread_ids_inverted[*]}")
    local grub_extra_cmdline="nohz_full=${thread_ids_str} \
    isolcpus=domain,managed_irq,${thread_ids_str} irqaffinity=${thread_ids_inverted_str}"
    grub_extra_cmdline=$(echo -n ${grub_extra_cmdline})

    echo ""
    echo "Generated configuration:"
    echo "${grub_extra_cmdline}"
    echo ""

    echo "Updating /etc/default/grub..."
    sed -i.bak -e "s|^#\?\(GRUB_CMDLINE_LINUX_DEFAULT\)=\"\(.*\)\"|\1=\"\2 ${grub_extra_cmdline}\"|g" /etc/default/grub

    echo "Running update-grub..."
    update-grub
    }

    main "${@}"
    EOF
  2. Reboot the server to apply the changes:

    reboot

  3. Log into the server under the user account you use to run Solana.

  4. Create the following script:

    cat << \EOF > ~/solana-affinity.sh
    #!/bin/bash

    # Process name
    PROCESS_NAME=agave-validator

    # Pinned CPU core
    SOLANA_POH_PINNED_CPU_CORE=2

    function main {
    # Check solana PID
    local solana_pid=$(pgrep -f "^${PROCESS_NAME}")
    if [[ -z $solana_pid ]]; then
    echo "Unable to retrieve main PID for process ${PROCESS_NAME}, exiting..." >&2
    fi

    # Check POH thread ID
    local poh_thread_id=$(ps -T -p "${solana_pid}" -o spid,comm | grep 'solPohTickProd' | awk '{print $1}')
    if [[ -z $poh_thread_id ]]; then
    echo "Unable to retrieve POH thread ID for process ${PROCESS_NAME}, exiting..." >&2
    fi

    # Check POH thread affinity
    local poh_thread_affinity=$(taskset -cp "${poh_thread_id}" 2>&1 | awk '{print $NF}')
    if [[ -z $poh_thread_affinity ]]; then
    echo "Unable to retrieve POH thread affinity for process ${PROCESS_NAME}, exiting..." >&2
    fi

    # Update POH thread affinity
    if [[ $poh_thread_affinity != $SOLANA_POH_PINNED_CPU_CORE ]]; then
    echo "Updating POH thread affinity for process ${PROCESS_NAME}..."
    taskset -cp ${SOLANA_POH_PINNED_CPU_CORE} "${poh_thread_id}"
    else
    echo "Thread affinity already applied for process ${PROCESS_NAME}, skipping update."
    fi
    }

    main "${@}"
    EOF
    chmod +x ~/solana-affinity.sh
  5. Open crontab editor:

    crontab -e

  6. Select the first option by pressing 1:

    (1. /bin/nano <---- easiest)

  7. Paste the following line at the end of the file:

    * * * * * ~/solana-affinity.sh >/dev/null 2>&1

How to check that binding worked fine

ps -U <your_user> -T -o pid,comm,psr | grep solPohTickProd

Correct answer should look like this after solana node finished syncing:

1451632 solPohTickProd 2

Have Questions?
Send us an email at [email protected]

Did this answer your question?