scst/scripts/run-regression-tests

#!/bin/bash

############################################################################
#
# Script for running those SCST regression tests that can be run automatically.
#
# Copyright (C) 2008-2020 Bart Van Assche <bvanassche@acm.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation, version 2
# of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
############################################################################

############################################################################
# This script performs the following actions:
# - Creates a temporary directory for storing the output of the regression
#   tests. No existing files are modified by this script.
# - Verifies whether the top-level *.patch files apply cleanly to the SCST
#   tree.
# - Duplicates the entire source tree to the temporary directory and
#   compiles the SCST source code.
# - Duplicates the entire source tree to the temporary directory, runs
#   'make 2release' and again compiles the SCST source code.
# - Checks whether the specified kernel version is present
#   in the directory specified through option -c.
# - If the source code of the specified kernel version is not present,
#   download it.
# - Convert the SCST source code into a kernel patch.
# - Extract the kernel sources.
# - Run checkpatch on the SCST kernel patch.
# - Apply the SCST kernel patch to the kernel tree.
# - Run 'make allmodconfig'.
# - Run the sparse source code checker on the SCST directory.
# - Run 'make headers_check'.
# - Compile the kernel tree.
# - Run 'make checkstack'.
# - Run 'make namespacecheck'.
# - Run 'make htmldocs'.
#
# Note: the results of the individual steps are not verified by this script
# -- the output generated by the individual steps has to be verified by
# reviewing the output files written into the temporary directory.
############################################################################


########################
# Function definitions #
########################

# shellcheck source=./kernel-functions
source "$(dirname "$0")/kernel-functions"

function usage {
  echo "Usage: $0 [-c <dir>] [-d <dir>] [-f] [-h] [-j <jobs>] [-p <patchdir>] [-q] <kver1> <kver2> ..."
  echo "        -c - cache directory for Linux kernel tarballs."
  echo "        -d - directory for temporary regression test files."
  echo "        -h - display this help information."
  echo "        -j - number of jobs that 'make' should run simultaneously."
  echo "        -k - remove temporary files before exiting."
  echo "        -q - download kernel sources silently."
  echo "        <kver1> <kver2> ... - kernel versions to test."
}

# Test whether the *.patch files in the SCST top-level directory apply cleanly
# to the SCST tree. Does not modify any files nor produce any output files.
function test_scst_tree_patches {
  local rc=0
  echo "Testing whether the SCST patches apply cleanly to the SCST tree ..."
  for p in *.patch srpt/patches/scst*.patch
  do
    if [ -e "$p" ] && ! patch -p0 -f --dry-run -s <"$p" &>/dev/null; then
      echo "ERROR: patch $p does not apply cleanly."
      rc=1
    fi
  done
  if [ "${rc}" = 0 ]; then
    echo "OK"
  fi
}

# Copy the entire SCST source code tree from "$1" into the current directory.
# Only copy those files which are administered by Subversion.
function duplicate_scst_source_tree {
  if [ -e "$1/AskingQuestions" ]; then
    "${scriptsdir}"/list-source-files "$1" |
      tar -C "$1" --files-from=- -c -f - |
      tar -x -f -
  else
    return 1
  fi
}

function make_rpm {
    local outputfile t

    for t in rpm scst-dkms-rpm; do
	outputfile="${outputdir}/make-${t}-output.txt"
	echo "Testing whether 'make ${t}' works fine ..."
	if make ${t} > "${outputfile}" 2>&1; then
	    echo "OK"
	else
	    echo "FAILED"
	fi
    done
}

function make_dpkg {
    local outputfile="${outputdir}/make-dpkg-output.txt"

    outputfile="${outputdir}/make-${t}-output.txt"
    echo "Testing whether 'make dpkg' works fine ..."
    if make dpkg > "${outputfile}" 2>&1; then
	echo "OK"
    else
	echo "FAILED"
    fi
}

function compile_scst {
  (
    for p in scst scst_local iscsi-scst srpt qla2x00t qla2x00t-32gbit; do
      if [ "${p%qla2x00t}" != "$p" ]; then
        BUILD_2X_MODULE=y CONFIG_SCSI_QLA_FC=y CONFIG_SCSI_QLA2XXX_TARGET=y \
        make -C "$p/qla2x00-target" || return $?
      else
        make -C "$p" || return $?
      fi
    done
  )
}

# Compile the unpatched SCST source code.
function compile_scst_unpatched {
  local scst="$PWD"
  local outputfile="${outputdir}/compilation-output-unpatched.txt"
  local workingdirectory="${outputdir}/scst-unpatched"

  echo "Testing whether the SCST tree compiles fine ..."
  (
    if mkdir -p "${workingdirectory}"  \
       && cd "${workingdirectory}"     \
       && duplicate_scst_source_tree "${scst}"  \
       && compile_scst &> "${outputfile}"
    then
      echo "OK"
    else
      echo "FAILED"
    fi
  )
}

# Compile the unpatched SCST source code without DLM.
function compile_scst_no_dlm {
  local scst="$PWD"
  local outputfile="${outputdir}/compilation-output-no-dlm.txt"
  local workingdirectory="${outputdir}/scst-no-dlm"

  echo "Testing whether the SCST tree compiles fine without DLM support ..."
  (
    if mkdir -p "${workingdirectory}"  \
       && cd "${workingdirectory}"     \
       && duplicate_scst_source_tree "${scst}"  \
       && CONFIG_SCST_NO_DLM=y compile_scst &> "${outputfile}"
    then
      echo "OK"
    else
      echo "FAILED"
    fi
  )
}

# Test out-of-tree compilation against the kernel header files in
# /lib/modules/$(uname -r)/build.
function compile_scst_patched {
  local scst="$PWD"
  local outputfile="${outputdir}/compilation-output-$1.txt"
  local workingdirectory="${outputdir}/scst-$1"

  echo "Testing whether the $1 SCST tree compiles fine ..."
  (
    if mkdir -p "${workingdirectory}"  \
       && cd "${workingdirectory}"     \
       && duplicate_scst_source_tree "${scst}"  \
       && make -s "$1"                 \
       && compile_scst &> "${outputfile}"
    then
      echo "OK"
    else
      echo "FAILED"
    fi
  )
}

# Generate a kernel patch from the SCST source tree for kernel version $1
# and with generate-kernel-patch options $2. $1 may be a
# ${kver}^${distro}^${release} triplet.
function generate_kernel_patch {
  local kver
  local patchfile="${outputdir}/scst-${1/^*}-kernel.patch"
  local patchfile_m="${outputdir}/scst-${1/^*}-kernel-matching-line-numbers.patch"
  local driver_options=""

  kver="$(kernel_version "${1/^*}")"
  driver_options=(-l)

  "${scriptsdir}"/generate-kernel-patch "${driver_options[@]}" $2 "$1" > "${patchfile}"

  "${scriptsdir}"/generate-kernel-patch "${driver_options[@]}" -n $2 "$1" > "${patchfile_m}"

  "${scriptsdir}"/generate-kernel-patch "${driver_options[@]}" -p "${outputdir}/${patchdir}" $2 "$1"
}

# Run checkpatch on the generated kernel patch. Assumes that there is a
# vanilla kernel tree present in directory "${outputdir}/linux-$1", and leaves
# this kernel tree clean.
function run_checkpatch  {
  local kver plevel errors warnings
  local outputfile="${outputdir}/checkpatch-$1-output.txt"
  local patchfile="${outputdir}/scst-$1-kernel.patch"

  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  if [ -e "${outputdir}/linux-$1/scripts/checkpatch.pl" ]; then
    if [ "${multiple_patches}" = "false" ]; then
      echo "Running checkpatch on the SCST kernel patch ..."
      ( cd "${outputdir}/linux-$1" \
        && scripts/checkpatch.pl --no-tree --no-signoff - < "${patchfile}" &> "${outputfile}")
    else
      echo "Running checkpatch on the SCST kernel patches ..."
      rm -f "${outputfile}"
      ( cd "${outputdir}/linux-$1" \
        && for p in "${outputdir}/${patchdir}"/*
           do
             echo "==== $p" >>"${outputfile}"
             scripts/checkpatch.pl --no-tree --no-signoff - < "$p" >> "${outputfile}" 2>&1
           done
      )
    fi
    errors=$(grep -c '^ERROR' "${outputfile}")
    warnings=$(grep -c '^WARNING' "${outputfile}")
    echo "${errors} errors / ${warnings} warnings."
    grep -E '^WARNING|^ERROR' "${outputfile}" |
      sort |
      grep -v 'WARNING: missing space after return type' |
      sed 's/^WARNING: Avoid CamelCase:.*/WARNING: Avoid CamelCase/' |
      uniq -c
  else
    echo "Skipping checkpatch step for kernel $1."
  fi
  return 0
}

function patch_and_configure_kernel {
  local kver
  kver="$(kernel_version "$1")"
  local patchfile="${outputdir}/scst-$1-kernel-matching-line-numbers.patch"
  local patchoutput="${outputdir}/patch-command-output-$1.txt"
  local configureoutput="${outputdir}/configure-output.txt"
  local disable="				\
CONFIG_BINARY_PRINTF				\
CONFIG_BLK_DEV_IO_TRACE				\
CONFIG_BRANCH_PROFILE_NONE			\
CONFIG_CONTEXT_SWITCH_TRACER			\
CONFIG_CTF					\
CONFIG_DEBUG_SECTION_MISMATCH			\
CONFIG_DEBUG_STRICT_USER_COPY_CHECKS		\
CONFIG_DTRACE					\
CONFIG_DT_CORE					\
CONFIG_DYNAMIC_FTRACE				\
CONFIG_EVENT_TRACE_TEST_SYSCALLS		\
CONFIG_FTRACE					\
CONFIG_FTRACE_MCOUNT_RECORD			\
CONFIG_FTRACE_NMI_ENTER				\
CONFIG_FTRACE_SELFTEST				\
CONFIG_FTRACE_STARTUP_TEST			\
CONFIG_FTRACE_SYSCALLS				\
CONFIG_FUNCTION_GRAPH_TRACER			\
CONFIG_FUNCTION_PROFILER			\
CONFIG_FUNCTION_TRACER				\
CONFIG_GENERIC_TRACER				\
CONFIG_HAVE_FTRACE_NMI_ENTER			\
CONFIG_HEADERS_CHECK				\
CONFIG_IRQSOFF_TRACER				\
CONFIG_IWLWIFI_DEVICE_TRACING			\
CONFIG_IWM_TRACING				\
CONFIG_KALLMODSYMS				\
CONFIG_KVM_MMU_AUDIT				\
CONFIG_MAC80211_DRIVER_API_TRACER		\
CONFIG_MMIOTRACE				\
CONFIG_NET_DROP_MONITOR				\
CONFIG_NOP_TRACER				\
CONFIG_RETPOLINE				\
CONFIG_RH_KABI_SIZE_ALIGN_CHECKS		\
CONFIG_SCHED_TRACER				\
CONFIG_SECURITY_SELINUX				\
CONFIG_STACK_TRACER				\
CONFIG_STACK_VALIDATION				\
CONFIG_TRACEPOINTS				\
CONFIG_TRACER_MAX_TRACE				\
CONFIG_TRACE_BRANCH_PROFILING			\
CONFIG_TRACING					\
CONFIG_UNWINDER_ORC				\
CONFIG_X86_32					\
CONFIG_X86_X32					\
"
  local enable="CONFIG_UNWINDER_FRAME_POINTER"

  case "$1" in
      2.6.32-*)
         echo "Enabling CONFIG_EVENT_TRACING for CentOS 6.x"
         enable="$enable \
                 CONFIG_DEBUG_KERNEL \
                 CONFIG_FTRACE \
                 CONFIG_EVENT_TRACING \
                 CONFIG_DYNAMIC_DEBUG";;
      *)
         disable="$disable CONFIG_EVENT_TRACING";;
  esac

  echo "Patching and configuring kernel ..."
  if [ "$ipv6" = "false" ]; then
      disable="$disable CONFIG_IPV6"
  fi
  (
    cd "${outputdir}/linux-$1" &&
    if [ "${multiple_patches}" = "false" ]; then
        if ! patch -p1 -f -s <"${patchfile}" >"${patchoutput}"; then
	    echo "Error: applying ${patchfile} failed."
	    exit 1
	fi
    else
        rm -f "${patchoutput}"
        for p in "${outputdir}/${patchdir}"/*; do
            echo "==== $p" >>"${patchoutput}"
            patch -p1 -f -s <"${p}" >>"${patchoutput}" 2>&1
        done
    fi &&
    make -s allmodconfig &>"${outputdir}/make-config-output.txt" &&
    for c in $disable; do sed -i.tmp "s/^$c=[ym]\$/$c=n/" .config; done &&
    for c in $enable; do sed -i.tmp "s/^\(# \)*$c\(=.*\| is not set\)\$/$c=y/" .config; done &&
    make -s oldconfig </dev/null &>"${configureoutput}" &&
    for c in $enable; do
	grep -q "^$c=[ym]" .config ||
	    echo "Enabling $c failed."
    done
  )
}

# Patches and compiles a kernel tree. Assumes that there is a vanilla kernel
# tree present in directory "${outputdir}/linux-$1".
function compile_patched_kernel {
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/kernel-$1-compilation-output.txt"

  echo "Compiling kernel $1 ..."
  (
    (
      cd "${outputdir}/linux-$1" \
      && LC_ALL=C make -s -k bzImage modules
    )
  ) &> "${outputfile}"
  echo "See also ${outputfile}."
  return 0
}

# Compile subdirectories $2..$n of the patched kernel tree linux-$1.
function compile_kernel {
  local kver plevel
  local k=$1
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/compilation-$1-output.txt"
  shift

  echo "Compiling the patched kernel ..."
  if (cd "${outputdir}/linux-$k" \
      && make -s modules_prepare \
      && make -s scripts \
      && for subdir; do LC_ALL=C make -k M="${subdir}"; done
     ) &> "${outputfile}"
  then
    local errors warnings
    errors=$(grep -c ' error:' "${outputfile}")
    warnings=$(grep -c ' warning:' "${outputfile}")
    echo "${errors} errors / ${warnings} warnings."
  else
    echo FAILED
  fi
  grep -E 'warning:|error:' < "${outputfile}" | sort | uniq -c
  return 0
}

# Run the source code verification tool 'sparse' on the SCST code. Assumes that
# there is a patched kernel tree present in directory "${outputdir}/linux-$1".
# For more information about endianness annotations, see also
# http://lwn.net/Articles/205624/.
function run_sparse {
  local k="$1"
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/sparse-$1-output.txt"
  shift

  echo "Running sparse on the patched kernel in $* ..."
  if (cd "${outputdir}/linux-$k" \
      && make -s modules_prepare \
      && make -s scripts \
      && if grep -q '^CONFIG_PPC=y$' .config; then LC_ALL=C make -k M=arch/powerpc/lib; fi \
      && for subdir; do
	  LC_ALL=C make -k C=2 CF="-D__CHECK_ENDIAN__ -DCONFIG_SPARSE_RCU_POINTER" M="${subdir}"
      done
    ) &> "${outputfile}"
  then
    local errors warnings
    errors=$(grep -c ' error:' "${outputfile}")
    warnings=$(grep -c ' warning:' "${outputfile}")
    echo "${errors} errors / ${warnings} warnings."
    grep -E ' warning:| error:' < "${outputfile}" \
      | sed -e 's/^[^:]*:[0-9:]* //' \
            -e "s/context imbalance in '[^']*':/context imbalance in <function>:/g" \
            -e "s/context problem in '[^']*': '[^']*'/context problem in <function>: <function>/g" \
            -e "s/function '[^']*'/function/g" \
            -e "s/symbol '[^']*'/symbol/g" \
      | sort \
      | uniq -c
  else
    echo FAILED
  fi

  return 0
}

# Run smatch for kernel version $1 on directories $2..$n
function run_smatch {
  local k="$1"
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/smatch-$1-output.txt"
  local disable="" # "CONFIG_DYNAMIC_DEBUG"
  shift

  echo "Running smatch on the patched kernel in $* ..."
  if (cd "${outputdir}/linux-$k" &&
      for c in $disable; do sed -i.tmp "s/^$c=y\$/$c=n/" .config; done &&
      make -s oldconfig </dev/null &&
      make -s modules_prepare &&
      make -s scripts &&
      if grep -q '^CONFIG_PPC=y$' .config; then LC_ALL=C make -k M=arch/powerpc/lib; fi &&
      for subdir; do
	LC_ALL=C make -k CHECK="smatch -p=kernel" C=2 CF=-D__CHECK_ENDIAN__ M="${subdir}"
      done
    ) &> "${outputfile}"
  then
    local errors warnings
    errors=$(grep -c ' error:' "${outputfile}")
    warnings=$(grep -c ' warning:' "${outputfile}")
    echo "${errors} errors / ${warnings} warnings."
    grep -E ' info:| warning:| error:' < "${outputfile}" |
      sed 's/^\([^:]*\):[0-9:]* /\1: /;s/\((see line \)[0-9]*\()\)/\1...\2/' |
      sort |
      uniq -c
  else
    echo FAILED
  fi

  return 0
}

function run_checkstack {
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/checkstack-$1-output.txt"

  echo "Running checkstack on the patched $1 kernel ..."
  (
    cd "${outputdir}/linux-$1" \
    && make -s modules_prepare \
    && make -s scripts \
    && LC_ALL=C make -k checkstack
  ) &> "${outputfile}"
  echo "See also ${outputfile}."
  return 0
}

function run_namespacecheck {
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/namespacecheck-$1-output.txt"

  echo "Running namespacecheck on the patched $1 kernel ..."
  (
    cd "${outputdir}/linux-$1" \
    && make -s modules_prepare \
    && make -s scripts \
    && LC_ALL=C make -k namespacecheck
  ) &> "${outputfile}"
  echo "See also ${outputfile}."
  return 0
}

function run_headers_check {
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/headers_check-$1-output.txt"

  echo "Running headers check on the patched $1 kernel ..."
  (
    cd "${outputdir}/linux-$1" \
    && make -s modules_prepare \
    && make -s scripts \
    && LC_ALL=C make -k headers_check
  ) &> "${outputfile}"
  local errors
  errors=$(grep -c '^[^ ]' "${outputfile}")
  echo "${errors} errors."
  grep '^[^ ]' "${outputfile}" | sed 's/.*: //' | sort | uniq -c
  return 0
}

function run_make_htmldocs {
  local kver plevel
  kver="$(kernel_version "$1")"
  plevel="$(patchlevel "$1")"
  local outputfile="${outputdir}/htmldocs-$1-output.txt"

  echo "Generating HTML documentation for the patched $1 kernel ..."
  (
    cd "${outputdir}/linux-$1" \
    && make -s modules_prepare \
    && make -s scripts \
    && LC_ALL=C make -k htmldocs
  ) &> "${outputfile}"
  echo "See also ${outputfile}."
  return 0
}


#########################
# Argument verification #
#########################

export LC_ALL=C

if [ ! -e scst ] || [ ! -e iscsi-scst ] || [ ! -e srpt ]; then
  echo "Please run this script from inside the SCST subversion source tree."
  exit 1
fi

scriptsdir="$(dirname "$0")"
if [ "${scriptsdir:0:1}" != "/" ]; then
  scriptsdir="$PWD/${scriptsdir}"
fi
# Where to store persistenly downloaded kernel tarballs and kernel patches.
kernel_sources="$HOME/software/downloads"
kernel_versions=""
# Directory in which the regression test output files will be stored. Must be
# an absolute path.
outputdir="${PWD}/regression-test-output-$(date +%Y-%m-%d_%Hh%Mm%Ss)"
# Driver configuration.
multiple_patches="false"
qla2x00t="true"
remove_temporary_files_at_end="false"
run_local_compilation="true"
quiet_download="false"

{
    # shellcheck disable=SC2046
    set -- $(/usr/bin/getopt "c:d:j:hklpq" "$@")
}
while [ "$1" != "${1#-}" ]
do
  case "$1" in
    '-c') kernel_sources="$2"; shift; shift;;
    '-d') outputdir="$2"; shift; shift;;
    '-h') usage; exit 1;;
    '-j') export MAKEFLAGS="-j$2"; shift; shift;;
    '-k') remove_temporary_files_at_end="true"; shift;;
    '-l') run_local_compilation="false"; shift;;
    '-p') multiple_patches="true"; shift;;
    '-q') quiet_download="true"; shift;;
    '--') shift;;
    *)    usage; exit 1;;
  esac
done

kernel_versions="$*"

if [ "${kernel_versions}" = "" ]; then
  usage
  exit 1
fi


####################
# Regression tests #
####################

if [ "$(type -p sparse)" = "" ]; then
  echo "Error: sparse has not yet been installed."
  echo "See also http://www.kernel.org/pub/software/devel/sparse/."
fi

if [ "$(type -p smatch)" = "" ]; then
  echo "Error: smatch has not yet been installed."
  echo "See also http://smatch.sourceforge.net/."
fi

if ! mkdir -p "${outputdir}"; then
  if [ -e "${outputdir}" ]; then
    echo "Error: directory ${outputdir} already exists."
  else
    echo "Error: could not create directory ${outputdir}."
  fi
fi

test_scst_tree_patches || exit $?
if [ "${run_local_compilation}" = "true" ]; then
  compile_scst_unpatched || exit $?
  compile_scst_no_dlm    || exit $?
  compile_scst_patched 2release || exit $?
  compile_scst_patched 2perf    || exit $?
  if type rpmbuild >/dev/null 2>&1; then
      make_rpm || exit $?
  fi
  if type debuild >/dev/null 2>&1; then
      make_dpkg || exit $?
  fi
fi

for kv in ${kernel_versions}
do
  echo "=========================="
  printf "= kernel %-15s =\n" "${kv}"
  echo "=========================="

  full_check="false"
  generate_kernel_patch_options=""
  ibmvio="false"
  run_checkpatch="true"
  run_sparse="true"
  run_smatch="true"
  ipv6="true"
  global_multiple_patches="${multiple_patches}"
  while true; do
    kv_without_opt="${kv%-?}"
    if [ "${kv_without_opt}" = "${kv}" ]; then
      kv_without_opt="${kv%-??}"
    fi
    kopt="${kv#${kv_without_opt}}"
    case "${kopt}" in
      '-4')  ipv6="false";;
      '-f')  full_check="true";;
      '-i')  ibmvio="true";;
      '-nc') run_checkpatch="false";;
      '-ns') run_sparse="false";;
      '-nm') run_smatch="false";;
      '-u')  generate_kernel_patch_options="-u";;
      '-p')  multiple_patches="true";;
      *)     break;;
    esac
    kv="${kv_without_opt}"
  done
  patchdir="patchdir-${kv}"
  k="${kv}"

  generate_kernel_patch "$k" "${generate_kernel_patch_options}" || continue
  (
      cd "${outputdir}" &&
	  download_and_extract_kernel_tree "$k"
  ) ||
      continue
  k="${k/^*}"
  if [ "${run_checkpatch}" = "true" ]; then
    run_checkpatch "$k"
  fi
  patch_and_configure_kernel "$k" || continue
  case "$k" in
      3.10.0-*)
	  # Make the CentOS 7.x build less noisy.
	  KCFLAGS+=" -Wno-deprecated-declarations";
  esac
  # See also commit bd664f6b3e37 ("disable new gcc-7.1.1 warnings for now";
  # v4.13) and commit 6f303d60534c ("gcc-9: silence 'address-of-packed-member'
  # warning"; v5.1).
  for w in -Wno-format-truncation -Wno-format-overflow -Wno-int-in-bool-context -Wno-address-of-packed-member; do
      if gcc -c -xc -E - "$w" </dev/null >&/dev/null; then
	  KCFLAGS+=" $w"
      fi
  done
  export KCFLAGS
  subdirs=(drivers/scst)
  if [ "${run_sparse}" = "true" ]; then
    run_sparse "$k" "${subdirs[@]}"
    mv "${outputdir}/sparse-$k-output.txt" \
       "${outputdir}/sparse-$k-scst-output.txt"
    if [ "${ibmvio}" = "true" ]; then
      run_sparse "$k" drivers/scsi libsrp.ko scsi_sysfs.ko \
                                 scsi_transport_fc.ko scsi_transport_srp.ko
      mv "${outputdir}/sparse-$k-output.txt" \
         "${outputdir}/sparse-$k-scsi-output.txt"
    fi
  fi
  if [ "${run_smatch}" = "true" ]; then
    run_smatch "$k" "${subdirs[@]}"
  fi
  if [ "${qla2x00t}" = "true" ] && kernel_version_le 2.6.37 "$k"; then
    subdirs+=(drivers/scsi/qla2xxx)
  fi
  compile_kernel "$k" "${subdirs[@]}"
  if [ "${full_check}" = "true" ]; then
    run_headers_check "$k"
    compile_patched_kernel "$k"
    run_checkstack "$k"
    run_namespacecheck "$k"
    run_make_htmldocs "$k"
  fi
  if [ "${remove_temporary_files_at_end}" = "true" ]; then
    rm -rf "${outputdir}"
    mkdir -p "${outputdir}"
  fi

  multiple_patches="${global_multiple_patches}"
done