Publish ascend-toolkit package

2026-04-11 13:50:33 +00:00
parent 897383c0c1
commit 4f8b599665
5 changed files with 448 additions and 0 deletions
--- a/flake.nix
+++ b/flake.nix
@@ -0,0 +1,125 @@
+{
+  description = "Ascend CANN Toolkit Nix package";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+
+        fetchAscendToolkit = pkgs.fetchurl {
+          url = "https://git.amadey.xyz/api/packages/mozart/generic/ascend-toolkit/8.5.0/ascend-toolkit-8.5.0-310p-full.tar.gz";
+          sha256 = "sha256-VqC+bG0k4eZDuQkTkPKgCvigUQM8QJU6ByVvj77sk+Q=";
+        };
+
+        ascend-toolkit = pkgs.stdenvNoCC.mkDerivation rec {
+          pname = "ascend-toolkit";
+          version = "8.5.0";
+
+          src = fetchAscendToolkit;
+
+          nativeBuildInputs = [
+            pkgs.patchelf
+            pkgs.autoPatchelfHook
+            pkgs.gcc.cc.lib
+            pkgs.glibc
+            pkgs.zlib
+            pkgs.sqlite
+          ];
+
+          phases = [ "unpackPhase" "installPhase" "fixupPhase" ];
+
+          unpackPhase = ''
+            mkdir -p src-copy
+            tar xzf "$src" -C src-copy
+          '';
+
+          installPhase = ''
+            mkdir -p $out
+            cp -r src-copy/* $out/
+            chmod -R u+w $out/
+            find $out -type f \( -name "bisheng" -o -name "bisheng-tune" \
+              -o -name "atc" -o -name "atc.bin" -o -name "akt" \
+              -o -name "ascendc_pack_kernel" -o -name "asc_dumper" \
+              -o -name "asc_opc" -o -name "msprof" -o -name "ccec" \
+              -o -name "cce-ld" -o -name "lld" -o -name "llvm-*" \
+              -o -name "*.so" -o -name "*.so.*" \) -exec chmod +x {} +
+            find $out/bin -type f -exec chmod +x {} + 2>/dev/null || true
+            find $out/tools -type f \( -name "*.py" -o -name "*.sh" \) -exec chmod +x {} + 2>/dev/null || true
+          '';
+
+          postFixup = ''
+            echo "Adding RPATH for toolkit-internal libraries..."
+            local rpath="$out/lib64:$out/devlib:$out/tools/aml/lib64"
+
+            for dir in bin tools/compiler x86_64-linux/bin x86_64-linux/ccec_compiler/bin \
+                       x86_64-linux/bisheng_compiler/bin; do
+              [ -d "$out/$dir" ] || continue
+              find "$out/$dir" -type f -executable 2>/dev/null | while IFS= read -r f; do
+                patchelf --add-rpath "$rpath" --force-rpath "$f" 2>/dev/null || true
+              done
+            done
+
+            find $out -type f \( -name "*.so" -o -name "*.so.*" \) 2>/dev/null | while IFS= read -r f; do
+              patchelf --add-rpath "$rpath" --force-rpath "$f" 2>/dev/null || true
+            done
+
+            local bisheng_config="$out/compiler/tikcpp/ascendc_kernel_cmake/legacy_modules/bisheng_config.cmake"
+            if [ -f "$bisheng_config" ]; then
+              cp ${./nix/bisheng_config_patched.cmake} "$bisheng_config"
+            fi
+          '';
+
+          passthru = {
+            setupHook = pkgs.makeSetupHook
+              { name = "ascend-toolkit-setup-hook.sh"; }
+              ./nix/ascend-toolkit-setup-hook.sh;
+          };
+        };
+
+        bisheng-wrapper = pkgs.stdenv.mkDerivation {
+          pname = "bisheng-wrapper";
+          version = "8.5.0";
+
+          dontUnpack = true;
+
+          nativeBuildInputs = [ pkgs.makeWrapper ];
+
+          installPhase = ''
+            mkdir -p $out/bin $out/nix-support
+
+            substituteAll ${./nix/bisheng-wrapper.sh} $out/bin/bisheng
+            chmod +x $out/bin/bisheng
+
+            substituteAll ${./nix/bisheng-wrapper.sh} $out/bin/bisheng-tune
+            chmod +x $out/bin/bisheng-tune
+
+            cat > $out/nix-support/add-flags.sh << 'EOF'
+            : "''${NIX_CFLAGS_COMPILE_bisheng:=''${NIX_CFLAGS_COMPILE:-}}"
+            : "''${NIX_CFLAGS_COMPILE_BEFORE_bisheng:=''${NIX_CFLAGS_COMPILE_BEFORE:-}}"
+            : "''${NIX_CFLAGS_LINK_bisheng:=''${NIX_CFLAGS_LINK:-}}"
+            : "''${NIX_LDFLAGS_bisheng:=''${NIX_LDFLAGS:-}}"
+            export NIX_CC_WRAPPER_FLAGS_SET_bisheng=1
+            EOF
+
+            touch $out/nix-support/add-local-cc-cflags-before.sh
+          '';
+
+          meta = with pkgs.lib; {
+            description = "Wrapper for Huawei Bisheng compiler that injects Nix compilation flags";
+            license = licenses.mit;
+          };
+        };
+      in
+      {
+        packages = {
+          inherit ascend-toolkit bisheng-wrapper;
+          default = ascend-toolkit;
+        };
+      }
+    );
+}
--- a/nix/ascend-toolkit-setup-hook.sh
+++ b/nix/ascend-toolkit-setup-hook.sh
@@ -0,0 +1,41 @@
+# Setup hook for ascend-toolkit
+# This is sourced when ascend-toolkit is in buildInputs of another derivation
+addAscendEnv() {
+    local toolkit="${1:?ascend-toolkit path required}"
+
+    # Core environment variables
+    export ASCEND_TOOLKIT_HOME="${toolkit}"
+    export ASCEND_HOME_PATH="${toolkit}"
+    export ASCEND_OPP_PATH="${toolkit}/opp"
+    export ASCEND_AICPU_PATH="${toolkit}"
+    export TOOLCHAIN_HOME="${toolkit}/toolkit"
+
+    # PATH
+    addToSearchPath PATH "${toolkit}/bin"
+    addToSearchPath PATH "${toolkit}/tools/ccec_compiler/bin"
+    addToSearchPath PATH "${toolkit}/tools/profiler/bin"
+    addToSearchPath PATH "${toolkit}/tools/ascend_system_advisor/asys"
+    addToSearchPath PATH "${toolkit}/tools/show_kernel_debug_data"
+    addToSearchPath PATH "${toolkit}/tools/msobjdump"
+
+    # LD_LIBRARY_PATH
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/lib64"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/lib64/plugin/opskernel"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/lib64/plugin/nnengine"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/opp/built-in/op_impl/ai_core/tbe/op_tiling/lib/linux/x86_64"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/tools/aml/lib64"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/tools/aml/lib64/plugin"
+    addToSearchPath LD_LIBRARY_PATH "${toolkit}/devlib"
+
+    # PYTHONPATH
+    addToSearchPath PYTHONPATH "${toolkit}/python/site-packages"
+    addToSearchPath PYTHONPATH "${toolkit}/opp/built-in/op_impl/ai_core/tbe"
+
+    # CMAKE_PREFIX_PATH
+    addToSearchPath CMAKE_PREFIX_PATH "${toolkit}/lib64/cmake"
+    addToSearchPath CMAKE_PREFIX_PATH "${toolkit}/toolkit/tools/tikicpulib/lib/cmake"
+}
+
+if [ -n "${ascend-toolkit:-}" ]; then
+    addAscendEnv "${ascend-toolkit}"
+fi
--- a/nix/bisheng-wrapper.sh
+++ b/nix/bisheng-wrapper.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+set -eu -o pipefail
+
+# Bisheng compiler wrapper - similar to NixOS cc-wrapper
+# This script injects NIX_CFLAGS_COMPILE and other Nix-managed flags
+
+if (( "${NIX_DEBUG:-0}" >= 7 )); then
+  set -x
+fi
+
+# Source flag accumulation from dependencies
+if [ -z "${NIX_CC_WRAPPER_FLAGS_SET_bisheng:-}" ]; then
+  if [ -f "@out@/nix-support/add-flags.sh" ]; then
+    source @out@/nix-support/add-flags.sh
+  fi
+fi
+
+# Build extra flags
+extraAfter=()
+extraBefore=()
+
+# Add sysroot for C standard library headers (critical for bisheng)
+ASCEND_TOOLKIT_HOME="${ASCEND_TOOLKIT_HOME:-}"
+if [ -n "$ASCEND_TOOLKIT_HOME" ] && [ -d "$ASCEND_TOOLKIT_HOME/tools/hcc/sysroot" ]; then
+  extraBefore+=("--sysroot=$ASCEND_TOOLKIT_HOME/tools/hcc/sysroot")
+fi
+
+# Add NIX_CFLAGS_COMPILE (the main flag we care about)
+if [ -n "${NIX_CFLAGS_COMPILE_bisheng:-}" ]; then
+  extraAfter+=($NIX_CFLAGS_COMPILE_bisheng)
+elif [ -n "${NIX_CFLAGS_COMPILE:-}" ]; then
+  extraAfter+=($NIX_CFLAGS_COMPILE)
+fi
+
+# Add NIX_CFLAGS_COMPILE_BEFORE if set
+if [ -n "${NIX_CFLAGS_COMPILE_BEFORE_bisheng:-}" ]; then
+  extraBefore+=($NIX_CFLAGS_COMPILE_BEFORE_bisheng)
+fi
+
+# Add linker flags if we're linking
+if [ -n "${NIX_CFLAGS_LINK_bisheng:-}" ]; then
+  extraAfter+=($NIX_CFLAGS_LINK_bisheng)
+fi
+
+# Convert LDFLAGS to -Wl, format
+if [ -n "${NIX_LDFLAGS_bisheng:-}" ]; then
+  for flag in $NIX_LDFLAGS_bisheng; do
+    extraBefore+=("-Wl,$flag")
+  done
+fi
+
+# Source local cflags hook if it exists
+if [ -f "@out@/nix-support/add-local-cc-cflags-before.sh" ]; then
+  source @out@/nix-support/add-local-cc-cflags-before.sh
+fi
+
+# Debug output
+if (( "${NIX_DEBUG:-0}" >= 1 )); then
+  echo "bisheng wrapper: extra flags before:" >&2
+  printf " %q\n" "${extraBefore[@]}" >&2
+  echo "bisheng wrapper: original flags:" >&2
+  printf " %q\n" "$@" >&2
+  echo "bisheng wrapper: extra flags after:" >&2
+  printf " %q\n" "${extraAfter[@]}" >&2
+fi
+
+# Find the real bisheng binary
+BISHENG_REAL="${BISHENG_REAL:-bisheng.real}"
+if ! command -v "$BISHENG_REAL" &>/dev/null; then
+  # Try to find bisheng in PATH (should be the real one from toolkit)
+  BISHENG_REAL="$(which bisheng 2>/dev/null || echo "bisheng")"
+fi
+
+# Execute the real bisheng compiler with injected flags
+exec "$BISHENG_REAL" \
+  "${extraBefore[@]+"${extraBefore[@]}"}" \
+  "$@" \
+  "${extraAfter[@]+"${extraAfter[@]}"}"
--- a/nix/bisheng_config_patched.cmake
+++ b/nix/bisheng_config_patched.cmake
@@ -0,0 +1,42 @@
+# ----------------------------------------------------------------------------------------------------------
+# Copyright (c) 2025 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+# ----------------------------------------------------------------------------------------------------------
+# Patched version: supports CMAKE_C/CXX_COMPILER override via environment variable
+string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSTEM_LOWER_PROCESSOR)
+if(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/x86_64-linux/ccec_compiler AND SYSTEM_LOWER_PROCESSOR STREQUAL "x86_64")
+    set(ASCENDC_DEVKIT_PATH ${ASCEND_CANN_PACKAGE_PATH}/x86_64-linux)
+elseif(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/aarch64-linux/ccec_compiler AND SYSTEM_LOWER_PROCESSOR STREQUAL "aarch64")
+    set(ASCENDC_DEVKIT_PATH ${ASCEND_CANN_PACKAGE_PATH}/aarch64-linux)
+elseif(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/tools/ccec_compiler)
+    set(ASCENDC_DEVKIT_PATH ${ASCEND_CANN_PACKAGE_PATH}/tools)
+elseif(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/compiler/ccec_compiler)
+    set(ASCENDC_DEVKIT_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler)
+else()
+    set(ASCENDC_DEVKIT_PATH ${ASCEND_CANN_PACKAGE_PATH}/ascendc_devkit)
+endif()
+
+
+set(CCEC_PATH           ${ASCENDC_DEVKIT_PATH}/ccec_compiler/bin)
+if(NOT "$ENV{CMAKE_C_COMPILER}" STREQUAL "")
+  set(CMAKE_C_COMPILER "$ENV{CMAKE_C_COMPILER}")
+else()
+  set(CMAKE_C_COMPILER    "${CCEC_PATH}/bisheng")
+endif()
+if(NOT "$ENV{CMAKE_CXX_COMPILER}" STREQUAL "")
+  set(CMAKE_CXX_COMPILER "$ENV{CMAKE_CXX_COMPILER}")
+else()
+  set(CMAKE_CXX_COMPILER  "${CCEC_PATH}/bisheng")
+endif()
+set(CMAKE_LINKER        "${CCEC_PATH}/ld.lld")
+set(CMAKE_AR            "${CCEC_PATH}/llvm-ar")
+set(CMAKE_STRIP         "${CCEC_PATH}/llvm-strip")
+set(CMAKE_OBJCOPY       "${CCEC_PATH}/llvm-objcopy")
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_SKIP_RPATH TRUE)
--- a/pack/pack_ascend_toolkit.sh
+++ b/pack/pack_ascend_toolkit.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+#
+# pack_ascend_toolkit.sh — создаёт два tar.gz архива из собранного ascend-toolkit:
+#
+#   1. ascend-toolkit-8.5.0-full.tar.gz    — полный архив (все файлы)
+#   2. ascend-toolkit-8.5.0-minimal.tar.gz — минимальный набор для режима sim
+#
+# Использование:
+#   ./pack_ascend_toolkit.sh [path_to_result_dir]
+#
+#   Если аргумент не указан, используется ./result
+
+set -euo pipefail
+
+RESULT_DIR="${1:-$(cd "$(dirname "$0")" && ls -d result 2>/dev/null || echo result)}"
+VERSION="8.5.0"
+OUTPUT_DIR="${2:-$(dirname "$0")}"
+
+if [ ! -d "$RESULT_DIR" ]; then
+    echo "[ERROR] Directory not found: $RESULT_DIR"
+    exit 1
+fi
+
+cd "$RESULT_DIR"
+
+# =============================================================================
+# 1. Полный архив — всё содержимое
+# =============================================================================
+echo "=== Creating FULL archive ==="
+FULL_ARCHIVE="${OUTPUT_DIR}/ascend-toolkit-${VERSION}-full.tar.gz"
+tar czf "$FULL_ARCHIVE" .
+FULL_SIZE=$(wc -c < "$FULL_ARCHIVE" | numfmt --to=iec-i --suffix=B)
+echo "  FULL archive: $FULL_ARCHIVE ($FULL_SIZE)"
+
+# =============================================================================
+# 2. Минимальный архив — только файлы для режима sim
+# =============================================================================
+echo "=== Creating MINIMAL archive (sim mode) ==="
+
+# Создаём временную директорию с минимальным набором
+MINIMAL_DIR=$(mktemp -d)
+trap "rm -rf $MINIMAL_DIR" EXIT
+
+# Копируем только нужные для sim-режима файлы/директории
+#
+# Структура основана на том, что реально используется при
+#   bash run.sh -r sim -v Ascend310P3
+#
+# Основные потребители:
+#   - cmake/npu_lib.cmake → compiler/tikcpp/ascendc_kernel_cmake
+#   - bisheng компиляция → tools/bisheng_compiler
+#   - cce линковка     → tools/ccec_compiler (через x86_64-linux/ccec_compiler symlink)
+#   - симулятор        → tools/simulator/Ascend310P3/lib
+#   - заголовки        → include
+#   - библиотеки       → lib64, devlib
+#   - OPP данные       → opp/built-in (нужны для op_impl)
+#   - Python bindings  → python/site-packages
+#   - Runtime          → runtime
+#   - Toolkit          → toolkit
+#   - Конфигурации     → conf, compat
+#   - set_env.sh       — корень
+# =============================================================================
+
+echo "  Copying essential files..."
+
+# Корневые файлы
+mkdir -p "$MINIMAL_DIR"
+cp -a set_env.sh "$MINIMAL_DIR/" 2>/dev/null || true
+
+# bin — содержит setenv.bash symlink + ключевые утилиты
+mkdir -p "$MINIMAL_DIR/bin"
+cp -a bin/* "$MINIMAL_DIR/bin/" 2>/dev/null || true
+
+# compiler — tikcpp cmake модули
+mkdir -p "$MINIMAL_DIR/compiler"
+cp -a compiler/tikcpp "$MINIMAL_DIR/compiler/" 2>/dev/null || true
+
+# compat
+mkdir -p "$MINIMAL_DIR/compat"
+cp -a compat/* "$MINIMAL_DIR/compat/" 2>/dev/null || true
+
+# conf
+mkdir -p "$MINIMAL_DIR/conf"
+cp -a conf/* "$MINIMAL_DIR/conf/" 2>/dev/null || true
+
+# devlib — stub библиотеки для разработки
+mkdir -p "$MINIMAL_DIR/devlib"
+cp -a devlib/* "$MINIMAL_DIR/devlib/" 2>/dev/null || true
+
+# fwkacllib — framework ACL библиотеки
+mkdir -p "$MINIMAL_DIR/fwkacllib"
+cp -a fwkacllib/* "$MINIMAL_DIR/fwkacllib/" 2>/dev/null || true
+
+# include — все заголовочные файлы (критично для компиляции)
+mkdir -p "$MINIMAL_DIR/include"
+cp -a include/* "$MINIMAL_DIR/include/" 2>/dev/null || true
+
+# lib64 — основные библиотеки
+mkdir -p "$MINIMAL_DIR/lib64"
+cp -a lib64/* "$MINIMAL_DIR/lib64/" 2>/dev/null || true
+
+# pkg_inc
+mkdir -p "$MINIMAL_DIR/pkg_inc"
+cp -a pkg_inc/* "$MINIMAL_DIR/pkg_inc/" 2>/dev/null || true
+
+# python
+mkdir -p "$MINIMAL_DIR/python"
+cp -a python/* "$MINIMAL_DIR/python/" 2>/dev/null || true
+
+# runtime
+mkdir -p "$MINIMAL_DIR/runtime"
+cp -a runtime/* "$MINIMAL_DIR/runtime/" 2>/dev/null || true
+
+# toolkit
+mkdir -p "$MINIMAL_DIR/toolkit"
+cp -a toolkit/* "$MINIMAL_DIR/toolkit/" 2>/dev/null || true
+
+# x86_64-linux — содержит архитектурно-зависимые файлы (symlinks на реальные директории)
+mkdir -p "$MINIMAL_DIR/x86_64-linux"
+cp -a x86_64-linux/* "$MINIMAL_DIR/x86_64-linux/" 2>/dev/null || true
+
+# tools — выборочно:
+#   - bisheng_compiler — основной AscendC компилятор
+#   - ccec_compiler   — linker для AI Core
+#   - simulator       — библиотека симулятора для sim режима
+#   - tikcpp          — tikcpp файлы
+#   - aml             — используется в LD_LIBRARY_PATH
+#   - profiler        — может использоваться
+#   - ascendc_tools   — ascendc инструменты
+mkdir -p "$MINIMAL_DIR/tools"
+cp -a tools/bisheng_compiler "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/ccec_compiler "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/simulator "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/tikcpp "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/aml "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/profiler "$MINIMAL_DIR/tools/" 2>/dev/null || true
+cp -a tools/ascendc_tools "$MINIMAL_DIR/tools/" 2>/dev/null || true
+
+# opp — operator packages
+# Для sim режима нужны op_impl (tiling и kernel реализации)
+mkdir -p "$MINIMAL_DIR/opp"
+cp -a opp/built-in "$MINIMAL_DIR/opp/" 2>/dev/null || true
+cp -a opp/include "$MINIMAL_DIR/opp/" 2>/dev/null || true
+cp -a opp/lib64 "$MINIMAL_DIR/opp/" 2>/dev/null || true
+
+echo "  Compressing..."
+MINIMAL_ARCHIVE="${OUTPUT_DIR}/ascend-toolkit-${VERSION}-minimal.tar.gz"
+tar czf "$MINIMAL_ARCHIVE" -C "$MINIMAL_DIR" .
+
+MINIMAL_SIZE=$(wc -c < "$MINIMAL_ARCHIVE" | numfmt --to=iec-i --suffix=B)
+echo "  MINIMAL archive: $MINIMAL_ARCHIVE ($MINIMAL_SIZE)"
+
+# =============================================================================
+# Итог
+# =============================================================================
+echo ""
+echo "=== Summary ==="
+echo "  FULL archive:    $FULL_SIZE"
+echo "  MINIMAL archive: $MINIMAL_SIZE"
+echo ""
+echo "Upload these files to your Gitea instance."
+echo "Then update flake.nix to use fetchurl instead of local path."