# Copyright 2020 Google LLC

#

# This source code is licensed under the BSD-style license found in the

# LICENSE file in the root directory of this source tree.

#

# Description:

# XNNPACK - optimized floating-point neural network operators library

load("@bazel_skylib//:bzl_library.bzl", "bzl_library")

load("@bazel_skylib//lib:selects.bzl", "selects") # buildifier: disable=out-of-order-load

load("@rules_python//python:py_binary.bzl", "py_binary")

load(

":build_defs.bzl",

"xnnpack_cc_library",

"xnnpack_cxx_library",

"xnnpack_if_kleidiai_enabled",

"xnnpack_min_size_copts",

"xnnpack_select_if",

"xnnpack_source_list_file",

"xnnpack_transitive_source_list",

"xnnpack_visibility",

)

load(

":build_params.bzl",

"xnnpack_archs",

"xnnpack_cc_library_for_arch",

"xnnpack_cond_for_arch",

"xnnpack_list_sum",

"xnnpack_simd_f16_archs",

"xnnpack_simd_f32_archs",

"xnnpack_simd_s16_archs",

"xnnpack_simd_s32_archs",

"xnnpack_simd_s8_archs",

"xnnpack_simd_u8_archs",

)

load(

"//gen:microkernels.bzl",

"all_srcs_for_arch",

"non_prod_asm_srcs_for_arch",

"non_prod_c_srcs_for_arch",

"prod_asm_srcs_for_arch",

"prod_c_srcs_for_arch",

)

load(

":build_srcs.bzl",

"LOGGING_SRCS",

"MICROKERNEL_DEFS",

"MICROKERNEL_HDRS",

"OPERATOR_SRCS",

"SUBGRAPH_SRCS",

"TABLE_SRCS",

)

package(default_visibility = ["//:__subpackages__"])

licenses(["notice"])

exports_files(["LICENSE"] + glob(["**/*.h"]))

exports_files(["preamble.js.lds"])

MICROKERNEL_DEPS = [

":assembly",

":common",

":intrinsics_polyfill",

":math",

":microkernel_hdrs",

":microparams_h",

":microparams_init",

":prefetch",

":sanitizers",

":simd_hdrs",

":tables",

":unaligned",

"//src/configs:hardware_config",

":logging",

]

SIMD_HEADERS = [

"src/xnnpack/simd/f32-avx-base.h",

"src/xnnpack/simd/f32-sse2-base.h",

"src/xnnpack/simd/f32-wasmsimd-base.h",

] + [

"src/xnnpack/simd/f32-" + arch + ".h"

for arch in xnnpack_simd_f32_archs()

] + [

"src/xnnpack/simd/f16-" + arch + ".h"

for arch in xnnpack_simd_f16_archs()

] + [

"src/xnnpack/simd/s16-" + arch + ".h"

for arch in xnnpack_simd_s16_archs()

] + [

"src/xnnpack/simd/s32-" + arch + ".h"

for arch in xnnpack_simd_s32_archs()

] + [

"src/xnnpack/simd/s8-" + arch + ".h"

for arch in xnnpack_simd_s8_archs()

] + [

"src/xnnpack/simd/u8-" + arch + ".h"

for arch in xnnpack_simd_u8_archs()

]

exports_files(SIMD_HEADERS)

ALL_MICROKERNEL_SRCS = xnnpack_list_sum(

[all_srcs_for_arch(arch) for arch in xnnpack_archs()],

[],

)

filegroup(

name = "microkernel_source_files",

data = ALL_MICROKERNEL_SRCS + ["src/microparams-init.c"],

visibility = xnnpack_visibility(),

)

filegroup(

name = "microkernel_header_files",

data = MICROKERNEL_HDRS + ["src/xnnpack/microparams.h"],

visibility = xnnpack_visibility(),

)

# Collection of internal microkernel related headers that declares all the microkernel functions.

xnnpack_cc_library(

name = "microkernel_hdrs",

hdrs = MICROKERNEL_HDRS,

deps = [

":common",

":math",

":microkernel_defs",

":microparams_h",

"//src/configs:config_hdrs",

],

)

xnnpack_cc_library(

name = "xnnpack_h",

hdrs = [

"include/experimental.h",

"include/xnnpack.h",

],

deps = [

":fingerprint_id",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "intrinsics_polyfill",

hdrs = ["src/xnnpack/intrinsics-polyfill.h"],

deps = [

":common",

":unaligned",

],

)

xnnpack_cc_library(

name = "microkernel_defs",

hdrs = MICROKERNEL_DEFS,

deps = [

":common",

":math",

],

)

xnnpack_cc_library(

name = "simd_hdrs",

hdrs = SIMD_HEADERS,

deps = [

":common",

":intrinsics_polyfill",

":math",

":unaligned",

],

)

xnnpack_cc_library(

name = "common",

hdrs = ["src/xnnpack/common.h"],

)

xnnpack_cc_library(

name = "init_once",

hdrs = ["src/xnnpack/init-once.h"],

deps = [

":common",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "microparams_h",

hdrs = [

"src/xnnpack/microparams.h",

],

deps = [

":common",

":math",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "microfnptr_h",

hdrs = [

"src/xnnpack/microfnptr.h",

],

deps = [

":common",

":math",

":microparams_h",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "params",

srcs = ["src/params.c"],

hdrs = [

"src/xnnpack/params.h",

],

deps = [

":common",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "fp16",

hdrs = ["src/xnnpack/fp16.h"],

)

xnnpack_cc_library(

name = "math",

hdrs = ["src/xnnpack/math.h"],

deps = [

":common",

":fp16",

],

)

xnnpack_cc_library(

name = "prefetch",

hdrs = ["src/xnnpack/prefetch.h"],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "unaligned",

hdrs = ["src/xnnpack/unaligned.h"],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "microparams_init",

srcs = ["src/microparams-init.c"],

hdrs = ["src/xnnpack/microparams-init.h"],

deps = [

":common",

":math",

":microparams_h",

":requantization",

":unaligned",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "node_type",

hdrs = [

"src/xnnpack/node-type.h",

"src/xnnpack/node-type-defs.inc",

],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "allocation_type",

hdrs = [

"src/xnnpack/allocation-type.h",

"src/xnnpack/allocation-type-defs.inc",

],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "operator_type",

hdrs = [

"src/xnnpack/operator-type.h",

"src/xnnpack/operator-type-defs.inc",

],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "microkernel_type",

hdrs = [

"src/xnnpack/microkernel-type.h",

"src/xnnpack/microkernel-type-defs.inc",

],

deps = [

":common",

],

)

xnnpack_cc_library(

name = "internal",

hdrs = ["src/xnnpack/internal.h"],

deps = [

":math",

":subgraph_h",

":xnnpack_h",

"//src/configs:config_hdrs",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "allocator",

srcs = [

"src/allocator.c",

],

hdrs = ["src/xnnpack/allocator.h"],

deps = [

":common",

":params",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "aligned_allocator",

hdrs = [

"src/xnnpack/aligned-allocator.h",

],

)

# TODO: This should be test_only = True.

xnnpack_cxx_library(

name = "buffer",

hdrs = ["src/xnnpack/buffer.h"],

deps = [

":common",

":datatype",

":logging",

":math",

":reference_ukernels",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "isa_checks",

hdrs = [

"src/xnnpack/isa-checks.h",

],

deps = [

":common",

"//src/configs:hardware_config",

],

)

xnnpack_cc_library(

name = "memory",

srcs = [

"src/memory.c",

],

hdrs = ["src/xnnpack/memory.h"],

# Override using xnnpack_gcc_std_copts, as it defines -std=c99, which then makes MAP_ANONYMOUS

# undefined

gcc_copts = [],

deps = [

":common",

":init_once",

":logging",

":math",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "quantization",

hdrs = ["src/xnnpack/quantization.h"],

deps = [

":math",

":microparams_h",

],

)

xnnpack_cc_library(

name = "requantization",

hdrs = ["src/xnnpack/requantization.h"],

deps = [

":common",

":math",

":microparams_h",

],

)

xnnpack_cc_library(

name = "requantization_stubs",

hdrs = ["src/xnnpack/requantization-stubs.h"],

)

xnnpack_cc_library(

name = "tables",

srcs = TABLE_SRCS,

deps = [

":common",

],

)

xnnpack_cc_library(

name = "assembly",

textual_hdrs = ["src/xnnpack/assembly.h"],

)

# Generate the prod/all microkernel libraries for each architecture.

[

[

xnnpack_cc_library_for_arch(

name = arch + "_prod_microkernels",

srcs = prod_c_srcs_for_arch(arch) + xnnpack_select_if(

":assembly_enabled",

prod_asm_srcs_for_arch(arch),

),

arch = arch,

# Internal build flag.

tags = ["nobuilder"],

deps = MICROKERNEL_DEPS,

),

xnnpack_cc_library_for_arch(

name = arch + "_all_microkernels",

srcs = non_prod_c_srcs_for_arch(arch) + xnnpack_select_if(

":assembly_enabled",

non_prod_asm_srcs_for_arch(arch),

),

arch = arch,

tags = ["nobuilder"],

deps = MICROKERNEL_DEPS + [":" + arch + "_prod_microkernels"],

),

]

for arch in xnnpack_archs()

]

xnnpack_cc_library(

name = "log_level_default",

defines = select({

# No logging in optimized mode

":optimized_build": ["XNN_LOG_LEVEL=0"],

# Warning logging in debug mode. Use `--define=xnn_log_level=debug` for full logging.

":debug_build": ["XNN_LOG_LEVEL=3"],

# Error-only logging in default (fastbuild) mode

"//conditions:default": ["XNN_LOG_LEVEL=2"],

}),

)

xnnpack_cc_library(

name = "logging",

srcs = LOGGING_SRCS,

hdrs = ["src/xnnpack/log.h"],

copts = select({

":debug_build": [],

"//conditions:default": xnnpack_min_size_copts(),

}) + select({

":xnn_log_to_stdio_explicit_true": ["-DXNN_LOG_TO_STDIO=1"],

"//conditions:default": [],

}),

defines = select({

":xnn_log_level_explicit_none": ["XNN_LOG_LEVEL=0"],

":xnn_log_level_explicit_fatal": ["XNN_LOG_LEVEL=1"],

":xnn_log_level_explicit_error": ["XNN_LOG_LEVEL=2"],

":xnn_log_level_explicit_warning": ["XNN_LOG_LEVEL=3"],

":xnn_log_level_explicit_info": ["XNN_LOG_LEVEL=4"],

":xnn_log_level_explicit_debug": ["XNN_LOG_LEVEL=5"],

"//conditions:default": [],

}),

visibility = xnnpack_visibility(),

deps = select({

":xnn_log_level_explicit_none": [],

":xnn_log_level_explicit_fatal": [],

":xnn_log_level_explicit_error": [],

":xnn_log_level_explicit_warning": [],

":xnn_log_level_explicit_info": [],

":xnn_log_level_explicit_debug": [],

"//conditions:default": [":log_level_default"],

}) + [

":allocation_type",

":common",

":microkernel_type",

":node_type",

":operator_type",

":xnnpack_h",

],

)

[

xnnpack_cc_library(

name = lib + "_microkernels",

deps = xnnpack_list_sum(

[

xnnpack_select_if(

xnnpack_cond_for_arch(arch),

[":" + arch + "_" + lib + "_microkernels"],

)

for arch in xnnpack_archs()

],

[],

),

)

for lib in ("prod", "all")

]

xnnpack_cc_library(

name = "indirection",

srcs = ["src/indirection.c"],

hdrs = ["src/xnnpack/indirection.h"],

deps = [

":common",

":math",

":microparams_h",

":operator_h",

":xnnpack_h",

"@FXdiv",

],

)

xnnpack_cxx_library(

name = "pack_lh",

srcs = ["src/pack-lh.cc"],

deps = [

":common",

":logging",

":math",

":microfnptr_h",

":microkernel_hdrs",

":microparams_h",

":quantization",

"//src/configs:config_hdrs",

"//src/configs:hardware_config",

] + xnnpack_if_kleidiai_enabled([

"@KleidiAI//kai/ukernels/matmul",

]),

)

xnnpack_cxx_library(

name = "packing",

srcs = ["src/reference/packing.cc"],

hdrs = ["src/xnnpack/pack.h"],

deps = [

":allocator",

":common",

":logging",

":math",

":microfnptr_h",

":microparams_h",

":microparams_init",

":unaligned",

":xnnpack_h",

"//src/configs:config_hdrs",

] + xnnpack_if_kleidiai_enabled([

"@KleidiAI//kai/ukernels/matmul",

]),

)

py_binary(

name = "generate_build_identifier_py",

srcs = ["scripts/generate-build-identifier.py"],

main = "scripts/generate-build-identifier.py",

tags = ["notap"], # This should only be built as part of the related genrule.

target_compatible_with = select({

"//build_config:emscripten": ["@platforms//:incompatible"],

"//conditions:default": [],

}),

)

# This target gathers the transitive sources from its deps and adds them to the

# srcs attribute.

xnnpack_transitive_source_list(

name = "build_identifier_ukernel_srcs",

srcs = ["src/reference/packing.cc"],

deps = [":prod_microkernels"],

)

# Windows builds fail on the generate_build_identifier genrule because the

# command line is too long when expanding the source file paths to feed to the

# Python generation script. To work around this, we write those files to a

# single file that we.

#

# This is what the below command does. It ONLY exposes the list file as an output.

#

# See the next genrule comment for the rest of the explanation.

xnnpack_source_list_file(

name = "build_identifier_ukernel_srcs_list",

srcs = [":build_identifier_ukernel_srcs"],

)

# When the python script gets executed, the files it has access to are limited

# to what is explicitely requested in the `srcs` attribute. We add the "list"

# and the source paths to the sources to make them available to the Python

# script and only expand the list file path to keep the command line short

# enough to avoid having Windows break down.

#

# I have not found a way to transitively pull the source files from the "list"

# target and make them available from the Python script without adding them to

# the `srcs` attribute (making them expanded in the "$(SRCS)" variable), thus

# the repetition.

genrule(

name = "generate_build_identifier",

srcs = [

":build_identifier_ukernel_srcs_list",

":build_identifier_ukernel_srcs",

],

outs = ["src/build_identifier.c"],

cmd = "$(location generate_build_identifier_py) --output $@ --input_file_list $(location :build_identifier_ukernel_srcs_list)",

tools = [":generate_build_identifier_py"],

)

xnnpack_cc_library(

name = "build_identifier",

srcs = ["src/build_identifier.c"],

)

xnnpack_cc_library(

name = "cache",

srcs = ["src/cache.c"],

hdrs = ["src/xnnpack/cache.h"],

deps = [

":allocator",

":common",

":logging",

":math",

":memory",

":mutex",

":xnnpack_h",

],

)

# Define a library with just the header to remove circular dependencies:

# operator-run (compute) <-> operators.

xnnpack_cc_library(

name = "compute_h",

hdrs = [

"src/xnnpack/compute.h",

],

deps = [

":common",

":microfnptr_h",

":microparams_h",

":operator_type",

":xnnpack_h",

"//src/configs:config_hdrs",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "microkernel_utils",

srcs = ["src/microkernel-utils.c"],

hdrs = ["src/xnnpack/microkernel-utils.h"],

deps = [

":common",

":logging",

":math",

"//src/configs:config_hdrs",

"//src/configs:hardware_config",

],

)

# Define a library with just the header to remove circular dependencies:

# indirection <-> operators.

xnnpack_cc_library(

name = "operator_h",

hdrs = [

"src/xnnpack/operator.h",

],

deps = [

":common",

":compute_h",

":microfnptr_h",

":microkernel_type",

":microparams_h",

":node_type",

":operator_type",

":xnnpack_h",

"//src/configs:config_hdrs",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "operator_utils",

srcs = ["src/operator-utils.c"],

hdrs = ["src/xnnpack/operator-utils.h"],

deps = [

":allocator",

":common",

":logging",

":math",

":microfnptr_h",

":node_type",

":operator_h",

":operator_type",

":params",

":xnnpack_h",

],

)

xnnpack_cxx_library(

name = "reference_ukernels",

srcs = [

"src/reference/binary-elementwise.cc",

"src/reference/unary-elementwise.cc",

],

hdrs = [

"src/xnnpack/reference-config.h",

"src/xnnpack/reference-utils.h",

],

deps = [

":common",

":datatype",

":math",

":microfnptr_h",

":microparams_h",

":xnnpack_h",

"//src/configs:config_hdrs",

],

)

xnnpack_cc_library(

name = "datatype",

srcs = ["src/datatype.c"],

hdrs = ["src/xnnpack/datatype.h"],

deps = [

":common",

":math",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "fingerprint_id",

srcs = ["src/operators/fingerprint_id.c"],

hdrs = [

"src/operators/fingerprint_id.h",

"src/operators/fingerprint_id.h.inc",

],

)

xnnpack_cc_library(

name = "fingerprint_cache",

srcs = ["src/operators/fingerprint_cache.c"],

hdrs = ["src/operators/fingerprint_cache.h"],

deps = [

":allocator",

":cache",

":fingerprint_id",

":init_once",

":mutex",

":operator_delete",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "fingerprint_check",

srcs = ["src/xnnpack/fingerprint_check.c"],

deps = [

":common",

":fingerprint_cache", # build_cleaner: keep

":fingerprint_id",

":internal",

":logging",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "operator_delete",

srcs = ["src/operator-delete.c"],

deps = [

":allocator",

":logging",

":operator_h",

":operator_utils",

":params",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "operators",

srcs = OPERATOR_SRCS,

hdrs = [

"src/xnnpack/compute.h",

"src/xnnpack/operator.h",

],

copts = select({

":debug_build": [],

"//conditions:default": xnnpack_min_size_copts(),

}) + select({

":xnn_enable_hmp_explicit_false": ["-DXNN_MAX_UARCH_TYPES=1"],

"//conditions:default": [],

}),

deps = [

":allocator",

":cache",

":common",

":datatype",

":fingerprint_cache",

":fingerprint_id",

":indirection",

":internal",

":logging",

":math",

":microfnptr_h",

":microkernel_hdrs",

":microkernel_type",

":microkernel_utils",

":microparams_h",

":microparams_init",

":node_type",

":normalization",

":operator_delete",

":operator_type",

":operator_utils",

":pack_lh",

":packing",

":params",

":quantization",

":reference_ukernels",

":subgraph_h",

":xnnpack_h",

"//src/configs:config_hdrs",

"//src/configs:hardware_config",

"//src/configs:microkernel_configs",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "sanitizers",

srcs = [

"src/sanitizers.c",

],

)

xnnpack_cc_library(

name = "subgraph_h",

hdrs = [

"src/xnnpack/subgraph.h",

],

deps = [

":allocation_type",

":common",

":math",

":node_type",

":xnnpack_h",

"//src/configs:config_hdrs",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "subgraph",

srcs = SUBGRAPH_SRCS,

hdrs = [

"src/subgraph/subgraph-utils.h",

"src/xnnpack/memory-planner.h",

"src/xnnpack/reshape-helpers.h",

"src/xnnpack/subgraph.h",

"src/xnnpack/subgraph-validation.h",

],

deps = [

":allocation_type",

":allocator",

":cache",

":common",

":datatype",

":fp16",

":internal",

":logging",

":math",

":memory",

":microkernel_hdrs",

":microkernel_type",

":mutex",

":node_type",

":operator_type",

":operator_utils",

":operators",

":params",

":requantization",

":subgraph_h",

":xnnpack_h",

"//src/configs:config_hdrs",

"//src/configs:hardware_config",

"//src/subgraph/rewrites:fp16_to_fp32",

"@pthreadpool",

],

)

xnnpack_cc_library(

name = "mutex",

srcs = [

"src/mutex.c",

],

hdrs = [

"src/xnnpack/mutex.h",

],

deps = [

":common",

":logging",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "normalization",

srcs = ["src/normalization.c"],

hdrs = [

"src/xnnpack/normalization.h",

],

deps = [

":common",

":math",

":xnnpack_h",

],

)

xnnpack_cc_library(

name = "XNNPACK",

srcs = select({

"//ynnpack/xnnpack:use_ynnpack": [],

"//conditions:default": ["src/init.c"],

}),

hdrs = [

"include/experimental.h",

"include/xnnpack.h",

],

copts = select({

":debug_build": [],

"//conditions:default": xnnpack_min_size_copts(),

}) + select({

":xnn_enable_hmp_explicit_false": ["-DXNN_MAX_UARCH_TYPES=1"],

"//conditions:default": [],

}),

visibility = xnnpack_visibility(),

deps = select({

"//ynnpack/xnnpack:use_ynnpack": ["//ynnpack/xnnpack"],

"//conditions:default": [

"//:allocator",

"//:build_identifier", # build_cleaner: keep

"//:common", # build_cleaner: keep

"//:fingerprint_check",

"//:fingerprint_id",

"//:init_once",

"//:logging",

"//:math", # build_cleaner: keep

"//:microkernel_hdrs", # build_cleaner: keep

"//:microparams_h", # build_cleaner: keep

"//:microparams_init", # build_cleaner: keep

"//:operators", # build_cleaner: keep

"//:params",

"//:subgraph", # build_cleaner: keep

"//src/configs:hardware_config",

"//src/configs:microkernel_configs", # build_cleaner: keep

"@pthreadpool",

],

}) + xnnpack_if_kleidiai_enabled([

"@KleidiAI//:common",

]),

)

############################# Build configurations #############################

# Enables usage of ARM FP16 (FP16 arithmetics) scalar kernels.

config_setting(

name = "xnn_enable_arm_fp16_scalar_explicit_true",

define_values = {"xnn_enable_arm_fp16_scalar": "true"},

)

# Disables usage of ARM FP16 (FP16 arithmetics) scalar kernels.

config_setting(

name = "xnn_enable_arm_fp16_scalar_explicit_false",

define_values = {"xnn_enable_arm_fp16_scalar": "false"},

)

# Enables usage of ARM FP16 (FP16 arithmetics) vector kernels.

config_setting(

name = "xnn_enable_arm_fp16_vector_explicit_true",