From 488ac3633963047b5a34093c03a4637c1a9a8a7e Mon Sep 17 00:00:00 2001 From: Radoslaw Wicik Date: Thu, 19 Nov 2020 09:17:30 +0100 Subject: [PATCH] [EGD-4341] Detect static globals in CI --- .github/workflows/main.yml | 4 + config/bootstrap_config | 4 +- in_docker.sh | 2 +- tools/find_global_data.py | 192 +++++++++++++++++++++++++++++++++++++ tools/find_global_data.sh | 53 +++++++++- 5 files changed, 248 insertions(+), 7 deletions(-) create mode 100755 tools/find_global_data.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aafffe56a795e288ab92b26dfed3f0d2bf5169c5..84466ca5a5638fe3650987089c3285ebe6f65474 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,8 +24,12 @@ jobs: pushd build-linux-Debug && \ export JOBS=${JOBS:-`nproc`} &&\ echo "JOBS=${JOBS}" &&\ + make -j ${JOBS} &&\ make -j ${JOBS} check && \ popd + - name: "Check for statics" + run: | + ./tools/find_global_data.py build-linux-Debug/PurePhone.elf - name: "Build for RT1051" run: | ./configure.sh rt1051 Release && \ diff --git a/config/bootstrap_config b/config/bootstrap_config index 989e81307ecc46cccbb091ba25cc5a4955cf96d8..a4363eb73b04e3c479964e06c8135bb7ac138a67 100644 --- a/config/bootstrap_config +++ b/config/bootstrap_config @@ -46,5 +46,7 @@ INSTALL_PACKAGES=" portaudio19-dev \ tzdata \ vim \ - wget + wget \ + python3-magic " + diff --git a/in_docker.sh b/in_docker.sh index f14bbe8b4efe621add43f6a78ef0d401bd7efc84..d2055c7bbf316ce496d73469ea66b5e8b07dda3b 100755 --- a/in_docker.sh +++ b/in_docker.sh @@ -3,7 +3,7 @@ # For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md CONTAINER_NAME="wearemudita/mudita_os_builder" -CONTAINER_TAG="latest" +CONTAINER_TAG="1.2" CONTAINER=${CONTAINER_NAME}:${CONTAINER_TAG} PURE_HOME=`pwd` STANDARD_OPTIONS="-v `pwd`:${PURE_HOME} --user \"$(id -u):$(id -g)\" --env HOME=${PURE_HOME} -t" diff --git a/tools/find_global_data.py b/tools/find_global_data.py new file mode 100755 index 0000000000000000000000000000000000000000..617e9c0ff9376e410b938b1e476f83f23b0c01c6 --- /dev/null +++ b/tools/find_global_data.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md + +detect static globals in code + +@package globalFinder +""" + +import argparse +import magic +import os +import re +import subprocess +import sys + +from enum import Enum +from queue import Queue +from threading import Thread + + +class ArchType(Enum): + """Define supported architecture types""" + + X86 = 1 + ARM = 2 + + +class Finder(): + """Finds potential places where global can be. + + This class holds all necessary variables and is responsible for holding and + presenting results. + + Attributes + ---------- + elfPath : str + path to inspected ELF + arch : ArchType + detected architecture + objdump : str + used `objdump` command (depends on arch) + nm : str + used `nm` command (depends on arch) + sections : Queue + stores list of sections with "StaticSymbol" + resultsQueue : Queue + stores found items, all threads are puting results here one result one item + results : dict + gathers results from `resultsQueue` used for counting items and printing report + """ + + armRe = re.compile("ELF 32-bit .*ARM.*") + x86Re = re.compile("ELF (64|32)-bit .*x86.*") + staticSymbol = "_Z41__static_initialization_and_destruction_0ii" + staticRe = re.compile("([0-9a-fA-F]+) ([0-9a-fA-F]+) .*" + staticSymbol + ".*") + + def __init__(self, path): + """ + Parameters + ---------- + path : str + path to elf that will be inspected + """ + + self.elfPath = path + self.arch = None + self.objdump = None + self.nm = None + self.sections = Queue() + self.resultsQueue = Queue() + self.results = {} + + def checkArch(self): + """Detect elf architecture""" + + magicString = magic.from_file(self.elfPath) + if self.x86Re.match(magicString): + self.arch = ArchType.X86 + self.objdump = "objdump" + self.nm = "nm" + elif self.armRe.match(magicString): + self.arch = ArchType.ARM + self.objdump = "arm-none-eabi-objdump" + self.nm = "arm-none-eabi-nm" + else: + self.arch = None + + def findStatic(self): + """find sections with static initialization symbol""" + + print("searching for statics") + out = subprocess.run([self.nm, "--print-size", self.elfPath], capture_output=True, text=True) + lines = out.stdout.split('\n') + for line in lines: + match = self.staticRe.match(line) + if match: + startAddress = int(match.group(1), 16) + sectionSize = int(match.group(2), 16) + endAddress = startAddress + sectionSize + self.sections.put(("0x%x" % startAddress, "0x%x" % endAddress)) + + def countResults(self): + """Count found items""" + + while not self.resultsQueue.empty(): + path, lineNo = self.resultsQueue.get() + realpath = os.path.realpath(path) + key = realpath + ":" + lineNo + if key not in self.results: + self.results[key] = 1 + else: + self.results[key] += 1 + + +class StaticChecker(Thread): + """Checks if found sections contain static symbols. + + This calass runs in separate processes objdump, deasembles section and search for hpp file. + if found such place is logged. + + Attributes + ---------- + + finder : Finder + access communication queues, and all other needed variables + hppRegexp : re + regexp for detecting hpp files + """ + + hppRegexp = re.compile("(.*\.hpp):([0-9]+)") + + def __init__(self, finder): + """ + Parameters + ---------- + + finder : Finder + Finder object to work with + """ + + Thread.__init__(self) + self.finder = finder + + def run(self): + while True: + print("-thread start:", self.native_id, self.finder.sections.qsize()) + if not self.finder.sections.empty(): + startAddress, stopAddress = self.finder.sections.get() + try: + out = subprocess.run([self.finder.objdump, + "--start-address="+startAddress, + "--stop-address="+stopAddress, + "-d", "-l", + self.finder.elfPath + ], + capture_output=True, text=True) + for line in out.stdout.split('\n'): + reMatch = self.hppRegexp.match(line) + if reMatch: + self.finder.resultsQueue.put((reMatch.group(1), reMatch.group(2))) + print(" ", reMatch.group(1), reMatch.group(2)) + finally: + self.finder.sections.task_done() + else: + return + + +def main(): + parser = argparse.ArgumentParser(description="Find globals in code") + parser.add_argument('elfPath', help="Path to elf file") + args = parser.parse_args() + finder = Finder(args.elfPath) + finder.checkArch() + finder.findStatic() + cpuCount = os.cpu_count() + if cpuCount is None: + cpuCount = 2 + for _ in range(cpuCount): + staticChecker = StaticChecker(finder) + staticChecker.daemon = True + staticChecker.start() + finder.sections.join() + finder.countResults() + if len(finder.results) >= 1: + print(finder.results) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/find_global_data.sh b/tools/find_global_data.sh index 329ca89e481679eb055e0a7e413800ba185d3b63..36f7ece86352d04f48d5f601893ce35efdc6ec57 100755 --- a/tools/find_global_data.sh +++ b/tools/find_global_data.sh @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/bash # Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. # For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md @@ -13,8 +13,51 @@ if [ ! -f ${objfile} ]; then exit 1 fi -arm-none-eabi-nm --print-size ${objfile} | grep _Z41__static_initialization_and_destruction_0ii | awk '{print $1" "$2}' | tr 'a-f' 'A-F' | while read -r symbol_start symbol_size; +type=$( file ${objfile} | grep "ARM" ) + +if [[ -z "${type}" ]]; then + echo "x86" + OBJDUMP="objdump" + NM="nm" +else + echo "ARM" + OBJDUMP="arm-none-eabi-objdump" + NM="arm-none-eabi-nm" +fi + +STATIC_SYMBOL="_Z41__static_initialization_and_destruction_0ii" + +staticList=($(${NM} --print-size ${objfile} | grep "${STATIC_SYMBOL}"|cut -f1,2 -d" "|tr " " ":" )) + +echo "${#staticList[@]}" + +I=0 +statcListSize=${#staticList[@]} + +searchRegexp='(.*\.hpp:[0-9]+)' +while [[ $I -lt $statcListSize ]] do - symbol_end=$(echo "obase=16;ibase=16;$symbol_start+$symbol_size" | bc) - arm-none-eabi-objdump --start-address=0x$symbol_start --stop-address=0x$symbol_end -d -l $objfile -done | grep ".hpp:" | awk '{print $1}' | xargs realpath | sed 's,^'${rootdir}/',,' | sort | uniq -c | sort -n + + symbol=${staticList[${I}]} + symbolStart="0x${staticList[${I}]%:*}" + symbolSize="0x${staticList[${I}]#*:}" + printf -v symbolEnd "0x%x" $(( ${symbolStart} + ${symbolSize} )) + echo "${I}/${statcListSize}: ${symbolStart} - ${symbolEnd}" + + items=( $(${OBJDUMP} --start-address=${symbolStart} --stop-address=${symbolEnd} -d -l ${objfile} ) ) + J=0 + while [[ ${J} -lt ${#items[@]} ]] + do + item=$( echo ${items[$J]} | cut -f1 -d" ") + if [[ ${item} =~ ${searchRegexp} ]]; then + paths="$paths\n${BASH_REMATCH[1]}" + fi + J=$(( $J + 1 )) + done + I=$(( $I + 1 )) +done + +if [[ -n ${paths} ]]; then + echo -e ${paths} | sort | uniq -c | sort -n +fi +