Compiler 2016: Testing Environement Specification - final

How a submission is judged in this phase

Boot a virtual machine in vbox of Ubuntu 14.04 with Oracle Java 1.8.0_77
git clone one's compiler2016
(IMPORTANT)git checkout final, the version with the 'final' tag is literally the most important one. Do push tags!
To judge this submission (e.g. abc/compiler2016/)
1. Kill all irrelevant processes
2. Setup a new bash process
3. Reset environment variables
4. cd abc/compiler2016/
5. make clean # delete the bin dir
6. make all # compile your code, please put your executables in a bin directory. A makefile is required!
7. source finalvars.sh # setup environment variables for mid-term testing. Define $CCHK in this script.
8. cd bin
9. For each test case data.mx
  1. $CCHK < data.mx > assem.s
  2. spim -stat -file assem.s <data.in >spimout
  3. compare data.out with spimout
  4. Don't output debug information to standard output in your submission. Please print your assembly only.

Notes

Do pay attention to Makefile!

Define $CCHK in your finalvars.sh

Testing Script

#!/bin/bash

echo 'please modify the $names variable to be your own id'
names=$(cat list.txt) #modify to be your own id
echo $names
sleep 2
timeo=180s

function _SC() #safe call
{
    echo "[RUNNING] : $*"
    eval $*
    if [ $? -ne 0 ]; then
        echo "[ERROR] : command $* failed, please check, exiting..."
        exit
    fi
}

outdir=$(pwd)

mkdir -p students

if [ ! -d compiler2016-testcases ]; then
    _SC git clone git@bitbucket.org:acmcompiler/compiler2016-testcases.git
else
    _SC cd compiler2016-testcases
    _SC git pull
    _SC cd ..
fi

normaldir=$(pwd)/compiler2016-testcases/passed
resdir=$(pwd)/results
mkdir -p $resdir
statistics=$resdir/statistics
#rm -rf $statistics

for name in ${names[@]}; do
    cd $outdir
    diff_result_dir=$resdir/diff_result_$name
    rm -rf $diff_result_dir
    mkdir -p $diff_result_dir
    log_file=$resdir/${name}.LOG
    rm -rf $log_file
    score=0
    full_score=0
    echo now testing ${name}...
    _SC cd students
    if [ ! -d $name ]; then
        _SC mkdir $name "&&" cd $name
        _SC git clone "git@bitbucket.org:${name}/compiler2016.git"
        _SC cd ..
    fi
    _SC cd $name/compiler2016
    _SC git checkout -f master
    _SC git pull 
    _SC git fetch --tags
    _SC git checkout -f final #pay attention to your tag! someone don't have the midterm tag
    _SC make clean #some one don't have make clean
    _SC make
    _SC cat ./finalvars.sh
 
    unset -v CCHK
    set -x
    source ./finalvars.sh #can't ensure SC here 
    set +x
    echo CCHK=$CCHK
 
    _SC cd bin 
    for filec in $(ls $normaldir/*.mx); do
        filein=${filec%.mx}.in
        fileout=${filec%.mx}.out
        _SC cp $filec data.mx
        timeout $timeo $CCHK <data.mx 1>assem.s
        if [ -f $filein ]; then 
            timeout $timeo spim -ldata 209715200 -lstack 104857600 -stat_file spimstat -file assem.s <$filein 1>spimout 2>/dev/null
        else
            timeout $timeo spim -ldata 209715200 -lstack 104857600 -stat_file spimstat -file assem.s 1>spimout 2>/dev/null
        fi
        pure_file_name=$(basename $filec)
        pure_file_name=${pure_file_name%.mx}
        diff spimout $fileout >$diff_result_dir/$pure_file_name
        full_score=$((full_score+1))
        file_limit=${filec%.mx}.limit
        if [ -f $file_limit ]; then
            full_score=$((full_score+1))
        fi
        if [ ! -s $diff_result_dir/$pure_file_name ]; then
            score=$((score+1))
            if [ -f $file_limit ]; then
                num=-1
                num=$(cat spimstat | awk ' { print $1 } ')
                limit=$(cat $file_limit)
                echo ${filec%.mx} : $num >>$log_file
                echo NUMBER: $num / LIMIT: $limit
                if [ $num -le $limit ]; then
                    score=$((score+1))
                fi
            fi
        else
            echo ${filec%.mx} : FAILED >>$log_file
            echo FAILED
        fi
        sleep 1
    done
    echo count: $score/$full_score >>$log_file
    echo count: $score/$full_score
    echo $name $score/$full_score >>$statistics
    _SC cd ..
    _SC git checkout -f master
    cd $outdir
done