Index: local/run.coco.sh =================================================================== diff -u -r1db270567b3e1fae4d91304b53deedeec6049e82 -r4294f85d44fb33898b6550015551dd8f67547013 --- local/run.coco.sh (.../run.coco.sh) (revision 1db270567b3e1fae4d91304b53deedeec6049e82) +++ local/run.coco.sh (.../run.coco.sh) (revision 4294f85d44fb33898b6550015551dd8f67547013) @@ -1,6 +1,4 @@ #!/bin/bash -# cmcsexeimport --csmes coco/tst_models/tst_models.csmes --title tst_models --csexe coco/tst_models/tst_models.csexe 2>coco/log/tst_models_cmcsexeimport.err | tee coco/tst_models/tst_models_cmcsexeimport.log -# cmreport --csmes=tst_models/tst_models.csmes --html=Report/index 2>coco/log/tst_models_cmreport.err | tee coco/tst_models/tst_models_cmreport.log ERR_TESTSUITE=10000 ERR_COCO_SERVER_CONNECTION=1001 @@ -16,14 +14,16 @@ APPLICATION_DIR=application TESTSUITE_DIR=testsuites +REPORT_TYPE="html" +COVERAGE_METHOD="" +COVERAGE_NAME="" WORKING_DIR=coco LOG_DIR=logs BUILD_DIR=build MERGE_DIR=merge REPORT_DIR=report DENALI_SQUISH=denaliSquish - PROJECT_PATH=$HOME/Projects WORKING_PATH=$HOME/coco REPORT_PATH=$WORKING_PATH/$REPORT_DIR @@ -34,39 +34,45 @@ APPLICATION_PATH=$PROJECT_PATH/$APPLICATION_DIR TESTSUITE_PATH=$PROJECT_PATH/$TESTSUITE_DIR - VCAN_SCRIPT=$PROJECT_PATH/scripts/setupCAN/setupVCAN.sh SQUISH_SERVER=/opt/squish-for-qt-7.1-20230505-1514/bin/squishserver SQUISH_RUNNER=/opt/squish-for-qt-7.1-20230505-1514/bin/squishrunner REPORT_COCO=$REPORT_PATH/coco REPORT_SQUISH=$REPORT_PATH/squish REPORT_QTTEST=$REPORT_PATH/qttest +SEPARATION_LINE="------------------------------------------------------------------------------" -##### ---------- HELPER ----------------------------------------------------------------------------------------------- +##################################################################################### +# Helper functions +##################################################################################### + function generate_alarm_mapping() { - echo " ------------------------------ Generating the alarms mapping" + local start_time=`date` + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Generating the alarms mapping - $start_time" 2>&1 | tee -a $SUMMARY_LOG + # alarmMapping must run in the application folder. pushd $APPLICATION_PATH # save current directory and switch to $APPLICATION_PATH /Projects/application ./alarmMapping.sh + + local end_time=`date` + elapsed_time "$start_time" "$end_time" "Alarms mapping completed - elapsed time" popd # restore previous directory } - -##### ---------- SERVER ----------------------------------------------------------------------------------------------- -##### ---------- SERVER COCO ------------------------------------------------------------------------------------------ +# Check coco server License and coco server connection function server_coco_check() { ping "$COCO_SERVER" -c 2 -W 1 if [ "$?" != "0" ]; then - echo " ------------------------------ Coco Lic Server unreachable" - exit $ERR_COCO_SERVER_CONNECTION + echo "*** Coco Lic Server unreachable ***" 2>&1 | tee -a $SUMMARY_LOG + exit $ERR_COCO_SERVER_CONNECTION fi } - -##### ---------- SERVER SQUISH ---------------------------------------------------------------------------------------- +# launch SERVER SQUISH function server_squish_start() { - echo " ------------------------------ Starting the Squish Server" + echo "Starting the Squish Server" 2>&1 | tee -a $SUMMARY_LOG # configuring the server $SQUISH_SERVER \ --config addAUT $DENALI_SQUISH \ @@ -75,12 +81,11 @@ | tee $LOG_PATH/squishqtserver.log # running the server - $SQUISH_SERVER & # must have '&' + $SQUISH_SERVER & # must have '&' } - function server_squish_run() { - echo " ------------------------------ Starting tests $TESTSUITE_PATH" + echo "Starting tests $TESTSUITE_PATH" 2>&1 | tee -a $SUMMARY_LOG $SQUISH_RUNNER \ --testsuite $TESTSUITE_PATH \ --reportgen html,$REPORT_SQUISH \ @@ -89,13 +94,13 @@ } function server_squish_stop() { - echo " ------------------------------ Stopping the Squish Server" + echo "Stopping the Squish Server" 2>&1 | tee -a $SUMMARY_LOG $SQUISH_SERVER --stop } - -##### ---------- INIT ------------------------------------------------------------------------------------------------- function cleanup_directories() { + echo "Removing directories..." 2>&1 | tee -a $SUMMARY_LOG + rm -frd $WORKING_PATH mkdir -p $WORKING_PATH mkdir -p $LOG_PATH @@ -106,35 +111,153 @@ mkdir -p $REPORT_QTTEST } +date_to_timestamp() { + date -d "$1" +"%s" +} -##### ---------- TESTS ------------------------------------------------------------------------------------------------ +function elapsed_time() { + start_timestamp=$(date_to_timestamp "$1") + end_timestamp=$(date_to_timestamp "$2") + + title="Elapsed time" + # Check if $3 has any value + if [ -n "$3" ]; then + title="$3" + fi + + time_difference=$((end_timestamp - start_timestamp)) + minutes=$((time_difference / 60)) + seconds=$((time_difference % 60)) + + echo "$title $minutes minutes and $seconds seconds" 2>&1 | tee -a $SUMMARY_LOG +} + + +function get_report_options(){ + REPORT_TYPE="html" + + # --coverage-mcc: Report multiple condition code coverage measurements. + # --coverage-mcdc: Report MC/DC coverage measurements. + # --coverage-condition: Report condition coverage measurements. + # --coverage-decision: Report decision coverage measurements. + # --line-coverage: Report line coverage measurements. + # --function-coverage: Report function coverage measurements. + + COVERAGE_NAME="" + COVERAGE_METHOD="" + + for param in "$@"; do + case $param in + "--html") + REPORT_TYPE="html" + ;; + "--csv") + REPORT_TYPE="csv" + ;; + "--mcc") + COVERAGE_METHOD=" --coverage-mcc " + COVERAGE_NAME="_mcc" + ;; + "--mcdc") + COVERAGE_METHOD=" --coverage-mcdc " + COVERAGE_NAME="_mcdc" + ;; + "--condition") + COVERAGE_METHOD=" --coverage-condition " + COVERAGE_NAME="_condition" + ;; + "--decision") + COVERAGE_METHOD=" --coverage-decision " + COVERAGE_NAME="_decision" + ;; + "--line") + COVERAGE_METHOD=" --line-coverage " + COVERAGE_NAME="_line" + ;; + "--function") + COVERAGE_METHOD=" --function-coverage " + COVERAGE_NAME="_function" + ;; + *) + # Handle unknown parameters or provide usage information + ;; + esac + done + + if [ -z "$COVERAGE_NAME" ]; then + echo "*** Warning *** No coverage method selected! Using default 'Condition coverage'" 2>&1 | tee -a $SUMMARY_LOG + COVERAGE_METHOD=" --coverage-condition " + COVERAGE_NAME="_condition" + fi +} + + +function copy_report_folder() { + # make a copy of the report folder using folder name with coverage emthod used + + if [ -d "$REPORT_PATH" ]; then + local new_name="$WORKING_PATH/report$1$COVERAGE_NAME" + # remove folder if it exists + if [ -d "$new_name" ]; then + rm -rf $new_name + fi + + echo "Copying $REPORT_PATH to new folder $new_name" 2>&1 | tee -a $SUMMARY_LOG + cp -r $REPORT_PATH $new_name + fi +} + +function usage(){ + echo "Usage:" + echo "run.coco.sh [options]" + echo " --build Build and compile the application" + echo " --run Run testsuite" + echo " --merge Merge test results" + echo " --report [type] [coverage method] Generate html or csv report" + echo " type options: " + echo " --html Default" + echo " --csv Comma separated value" + echo " coverage method options:" + echo " --condition (Default) - Report condition coverage measurements" + echo " --mcc Report multiple condition code coverage measurements" + echo " --mcdc Report MC/DC coverage measurements" + echo " --decision Report decision coverage measurements" + echo " --line Report line coverage measurements" + echo " --function Report function coverage measurements" + echo "" + echo " --reports [type] [coverage method] Create seperate reports" + echo " --all [type] [coverage method] Perform all phases: Build, run, merge, and reports" +} + +##################################################################################### +# Test functions +##################################################################################### + function tests_copy_list() { cp $QT_TESTS_LIST_PATH/$QT_TESTS_LIST_FILE $WORKING_PATH } - function tests_read_list() { QT_TESTS_LIST=() # initialize empty array # read a file $QT_TESTS_LIST_FILE. read each line - echo " ------------------------------ Test List:" + echo "Test List:" 2>&1 | tee -a $SUMMARY_LOG while read -r line do # remove leading spaces, remove lines that starts with #, store result in test test=`echo $line | sed -e 's/^[[:space:]]*//' | sed -e '/^#/d'` # if test is not empty then add it to the array if [ "$test" != "" ]; then - QT_TESTS_LIST+=($test) - echo " - $test" + QT_TESTS_LIST+=($test) + echo " - $test" 2>&1 | tee -a $SUMMARY_LOG fi done < "$QT_TESTS_LIST_FILE" } - function tests_setup() { if [ $($VCAN_SCRIPT) -eq 0 ];then - echo " ------------------------------ Canbus setup failed" + echo "*** Canbus setup failed ***" 2>&1 | tee -a $SUMMARY_LOG exit $ERR_CANBUS_SETUP fi } @@ -155,13 +278,15 @@ done } -##### ---------- COVERAGE --------------------------------------------------------------------------------------------- function coverage_build() { - echo " ------------------------------ Run qmake" + local start_time=`date` + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Building - $1 - $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo $(pwd) $QT_PATH/qmake \ - $APPLICATION_PATH/denali.pro \ + $APPLICATION_PATH/denali.pro \ -spec linux-g++ \ CONFIG+="$1" \ CONFIG+=CodeCoverage \ @@ -174,13 +299,23 @@ | tee "$LOG_PATH/build_$1".log /usr/bin/make clean rm *.csmes - echo " ------------------------------ Cleaned for next build" + + local end_time=`date` + elapsed_time "$start_time" "$end_time" "Building completed - elapsed time" } - function coverage_merge() { - # MMSS 1 Get all csmes reports from each test case - echo " ------------------------------ Merging the excecution report databases" + ########################################################################################### + # Use utility "cmmerge" to merge several instrumentation databases (.csmes file) together + # It behaves like the CoverageBrowser + # cmmerge -o ... + ########################################################################################### + + # Get all csmes reports from each test case + local start_time=`date` + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Merging the excecution report databases - $start_time" 2>&1 | tee -a $SUMMARY_LOG + local csmes="" for test in ${QT_TESTS_LIST[@]};do csmes+="$BUILD_PATH/$test/$test.csmes " @@ -191,11 +326,16 @@ $csmes \ 2> $LOG_PATH/$MERGE_DIR.err \ | tee $LOG_PATH/$MERGE_DIR.log + + local end_time=`date` + elapsed_time "$start_time" "$end_time" "Merging completed - elapsed time" } - function coverage_import() { - echo " ------------------------------ Importing the execution report into the coverage database" + local start_time=`date` + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Importing execution report into coverage database - $start_time" 2>&1 | tee -a $SUMMARY_LOG + local csexe="" for test in ${QT_TESTS_LIST[@]};do echo $BUILD_PATH/$test/$test.csexe @@ -206,135 +346,250 @@ 2> $LOG_PATH/import_$test.err \ | tee $LOG_PATH/import_$test.log done + + local end_time=`date` + elapsed_time "$start_time" "$end_time" "Importing completed - elapsed time" } - function coverage_report() { - REPORT_TYPE="html" + ################################################################################################ + # cmreport – Code coverage report generation utilityt + # + # Coverage method options + # --coverage-mcc: Report multiple condition code coverage measurements. + # --coverage-mcdc: Report MC/DC coverage measurements. + # --coverage-condition: Report condition coverage measurements. + # --coverage-decision: Report decision coverage measurements. + # --line-coverage: Report line coverage measurements. + # --function-coverage: Report function coverage measurements. + ############################################################################################### + + #REPORT_TYPE="html" REPORT_CMD="--html=$REPORT_COCO/index" - if [[ "$1" == "csv" ]]; then + if [[ "$REPORT_TYPE" == "csv" ]]; then REPORT_TYPE="csv" REPORT_FILE=report_out.csv REPORT_CMD="--csv-excel=$REPORT_FILE" fi - echo " ------------------------------ Generating the coverage report <$REPORT_TYPE>" - cmreport \ - --csmes=$MERGE_PATH/$MERGE_DIR.csmes \ - $REPORT_CMD \ - 2> $LOG_PATH/report.err \ + + local start_time=`date` + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Generating coverage report - $start_time" 2>&1 | tee -a $SUMMARY_LOG + + cmreport \ + $COVERAGE_METHOD \ + --csmes=$MERGE_PATH/$MERGE_DIR.csmes \ + $REPORT_CMD \ + 2> $LOG_PATH/report.err \ | tee $LOG_PATH/report.log - if [[ "$1" == "csv" ]]; then + if [[ "$REPORT_TYPE" == "csv" ]]; then mv report_out.csv report/ fi - echo " ------------------------------ Out file location : $(pwd)/$REPORT_COCO/$REPORT_FILE" + + local end_time=`date` + echo "Single report generated: $(pwd)/$REPORT_COCO/$REPORT_FILE - $end_time" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "" + echo "" } - -function coverage_reports() { - echo "processing separate execution reports" > $LOG_PATH/$MERGE_DIR.err | tee $LOG_PATH/$MERGE_DIR.log +function separate_coverage_reports() { + echo "Processing separate execution reports" > $LOG_PATH/$MERGE_DIR.err | tee $LOG_PATH/$MERGE_DIR.log - ############################################################################################# - # - # in each Loop eteration do the following: - # 1) get csmes value - # 2) Using $scmes variable execute coverage import command "cmcsexeimport" - # 3) Using $cscmes execute coverage report command "cmreport" - # - ########################################################################################### + ################################################################################## + # Each Loop iteration do: + # 1) Get csmes value + # 2) Execute coverage using coco command "cmcsexeimport" + # 3) Execute coverage report using coco command "cmreport" + ################################################################################## for test in ${QT_TESTS_LIST[@]};do + local start_time=`date` csmes="$BUILD_PATH/$test/$test.csmes" # coverage import echo $BUILD_PATH/$test/$test.csexe cmcsexeimport --csmes=$csmes --title==$test --csexe=$BUILD_PATH/$test/$test.csexe \ - 2>> $LOG_PATH/import_$test.err | tee $LOG_PATH/import_$test.log + 2> $LOG_PATH/import_$test.err | tee $LOG_PATH/import_$test.log fname=$(basename "$csmes") # remove path REPORT_FILE="${fname%.*}" # remove extension - REPORT_TYPE="html" + #REPORT_TYPE="html" REPORT_CMD="--html=$REPORT_COCO/$REPORT_FILE" # .html - echo "--------- Generating report $REPORT_FILE" + echo $SEPARATION_LINE 2>&1 | tee -a $SUMMARY_LOG + echo "Generating report ${REPORT_FILE}.${REPORT_TYPE} - $start_time" 2>&1 | tee -a $SUMMARY_LOG - if [[ "$1" == "csv" ]]; then + if [[ "$REPORT_TYPE" == "csv" ]]; then REPORT_TYPE="csv" REPORT_FILE="${REPORT_FILE}.csv" REPORT_CMD="--csv-excel=$REPORT_FILE" fi - echo " ------------------------------ Generating the coverage report <$REPORT_TYPE>" - - cmreport --csmes=$csmes $REPORT_CMD 2>> $LOG_PATH/report.err | tee $LOG_PATH/report.log + echo "########### ########### $COVERAGE_METHOD #########" + + + #cmreport $COVERAGE_METHOD --csmes=$csmes $REPORT_CMD 2> $LOG_PATH/report.err | tee $LOG_PATH/report.log + + cmreport \ + $COVERAGE_METHOD --csmes=$csmes $REPORT_CMD \ + 2> $LOG_PATH/report.err | tee $LOG_PATH/report.log + - if [[ "$1" == "csv" ]]; then + if [[ "$REPORT_TYPE" == "csv" ]]; then mv report_out.csv report/ fi - - echo " ------------------------------ Out file location : $(pwd)/$REPORT_COCO/$REPORT_FILE" + + local end_time=`date` + echo "Report generation completed - `date` " 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "" + echo "" done } +##################################################################################### +# Application phases based on optional parameter +##################################################################################### -##### ---------- MAIN ------------------------------------------------------------------------------------------------- - -if [ "$1" == "--build" ]; then +function build_phase() { server_coco_check cleanup_directories + local start_time=`date` + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG + echo "Starting Build Phase $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG cd $WORKING_PATH + tests_copy_list tests_read_list generate_alarm_mapping + for test in ${QT_TESTS_LIST[@]};do coverage_build $test done -fi + + local end_time=`date` + echo "*** Build phase completed *** `date`" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "Build phase elapsedd time" +} - -if [ "$1" == "--run" ]; then +function run_phase() { + touch $LOG_PATH/sun_summary.log + local start_time=`date` + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG + echo "Starting Run Phase $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG cd $WORKING_PATH tests_read_list tests_run -fi + local end_time=`date` + echo "*** Run phase completed *** `date`" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "Run phase elapsed time" +} - -if [ "$1" == "--merge" ]; then +function merge_phase() { + local start_time=`date` + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG + echo "Starting Merge Phase $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG server_coco_check cd $WORKING_PATH tests_read_list # generate Q_TESTS_LIST coverage_merge coverage_import -fi + local end_time=`date` + echo "*** Merge phase completed *** `date`" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "Merge phase elapsed time" +} - -if [ "$1" == "--report" ]; then +function single_report_phase() { + local start_time=`date` + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG + echo "Generating SingleReport Phase $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo "Coverage Method: $COVERAGE_METHOD" 2>&1 | tee -a $SUMMARY_LOG + echo "Report Type: $REPORT_TYPE" 2>&1 | tee -a $SUMMARY_LOG + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG server_coco_check cd $WORKING_PATH coverage_report "$2" -fi + local end_time=`date` + echo "*** Single global report phase completed *** `date`" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "Single report generation phase elapsed time " +} - -if [ "$1" == "--reports" ]; then +function separate_report_phase() { + local start_time=`date` + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG + echo "Generating Separte Reports Phase $start_time" 2>&1 | tee -a $SUMMARY_LOG + echo "Coverage Method: $COVERAGE_METHOD" 2>&1 | tee -a $SUMMARY_LOG + echo "Report Type: $REPORT_TYPE" 2>&1 | tee -a $SUMMARY_LOG + echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG server_coco_check cd $WORKING_PATH tests_read_list # generate Q_TESTS_LIST - - coverage_reports "$2" + separate_coverage_reports "$2" + local end_time=`date` + echo "*** Separte Reports phase completed *** `date`" 2>&1 | tee -a $SUMMARY_LOG + elapsed_time "$start_time" "$end_time" "Seperate report generation phase elapsed time" +} + +##################################################################################### +# M a i n E n t r y +##################################################################################### + +clear + +# if option --help or no parameters are provided then usage will appear and exit the scrpt +if [ "$1" == "" ] || [ "$1" == "--help" ]; then + usage + exit 0 fi +# Check if the SUMMARY_LOG file exists, create it +# summary log provides brief description of script task being executed, timestamp, and elapsed time +# summary.log will be recreated when build option is triggered, else entries will be appened +SUMMARY_LOG="$LOG_PATH/summary.log" +if [ ! -f "$SUMMARY_LOG" ]; then + # Create the log file with a time and date header + echo "###########################################################################" > "$SUMMARY_LOG" + echo "Summary Log - created on $(date)" >> "$SUMMARY_LOG" + echo "###########################################################################" >> "$SUMMARY_LOG" + echo " " >> "$SUMMARY_LOG" +fi -if [ "$1" == "" ]; then - echo "Usage:" - echo "run.coco.sh " - echo "--build Build and instrment the application and tests from $APPLICATION_PATH" - echo "--run run the tests" - echo "--merge merge the test results" - echo "--report Create the execution report of the ran applicaiton" - echo " Report type: html:default, csv" - echo "--reports Create seperate reports of the run applicaiton" - echo " Report type html:default, csv" +app_start_time=`date` # to calculate entire script execution elapsed time + +if [ "$1" == "--build" ]; then + build_phase +elif [ "$1" == "--run" ]; then + run_phase +elif [ "$1" == "--merge" ]; then + merge_phase +elif [ "$1" == "--report" ]; then + get_report_options "$@" # report Type, coverage_method + single_report_phase + copy_report_folder "_single" # make a copy of the report folder using coveerage method used +elif [ "$1" == "--reports" ]; then + get_report_options "$@" # report Type, coverage_method + separate_report_phase + copy_report_folder "_multi" # make a copy of the report folder using coveerage method used +elif [ "$1" == "--all" ]; then + get_report_options "$@" # report Type, coverage_method + build_phase + run_phase + merge_phase + single_report_phase + separate_report_phase + copy_report_folder "_all" # make a copy of the report folder using coveerage method used +else + usage + exit 1 fi +echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG +echo "End of script - $(date)" 2>&1 | tee -a $SUMMARY_LOG +echo "###########################################################################" 2>&1 | tee -a $SUMMARY_LOG +app_end_time=`date` +elapsed_time "$app_start_time" "$app_end_time" "Total phase(s) elapsed time"