#!/bin/bash
exit_with_error() { echo "ERROR: $*"; exit 1; }
verify_defined() { eval val='$'$1; [ -z "$val" ] && exit_with_error "$1 not defined"; }

set -x
if [[ $1 == "-m" ]]; then 
    # manual run by admin, i.e.: CW_INTERVAL=1440 ./corewatcher.sh -m
    set -e
    ISS_ENV=$(mktemp /tmp/cw_$$_env.XXX)
    echo "export BSCONFIG_IDIR='$(pwd)'" > $ISS_ENV
    cat metainstance.json | python -c 'import sys, json; dict=json.load(sys.stdin);'$'\n''for kv in dict[u"leader"][u"properties"].iteritems(): print "export {}=\"{}\"".format(kv[0],kv[1]);' >> $ISS_ENV
    source $ISS_ENV
    rm $ISS_ENV
    set +e
    shift 1
else
    trap 'rm -f /tmp/cw_$$_*.??? 2>/dev/null' EXIT
fi


# For DM-based services: get service name from config, use binary name otherwise
get_key() {
    RESULT=""
    ATTEMP=10
    while [ 1 ]
    do
        if [ -r $C_PATH ]
        then
            RESULT=`egrep -o "(\w+-?)+" $C_PATH | tr "\n" " " | egrep -o "$1 (\w+-?)+" | cut -d" " -f2;`
        fi
        if [ "x$RESULT" != "x" ]
        then
            echo -n $RESULT
            return 0
        fi
        ATTEMP=$(($ATTEMP - 1))
        if [ $ATTEMP -eq "0" ]
        then
            echo -n $2
            return 0
        fi
        sleep 1
    done
}

# For non-DM services: use ISS variables to determine service name
get_tag() {
    RESULT=$(echo $BSCONFIG_ITAGS | grep -oP '(?<=a_'"$1"'_)[^ _]*')
    [[ $RESULT == "unknown" ]] && RESULT=
    [[ -z $RESULT ]] && RESULT=$2
    echo -n $RESULT
}

get_service_name() {
    RESULT=`get_tag prj`
    [[ -z $RESULT ]] && RESULT=$NANNY_SERVICE_ID
    [[ -z $RESULT ]] && RESULT=$2
    echo -n $RESULT
}

# Mailing routine (enabled if CW_EMAILS is set)
mail_report_to_saas() {
    # Constants
    MARKER='----==--boundary'
    B_START='--'${MARKER}'\nContent-Type: application/x-gzip; name="REPLACE_IT"\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="REPLACE_IT"\n'
    B_END=${MARKER}'--'
    SUBJ="Coredump for ${CTYPE}:${S_NAME} on `hostname`"

    trace_file=$1
    [[ -f $trace_file ]] || exit_with_error Trace file missing

    cod_path=$2
    report=`mktemp /tmp/cw_$$_report.XXX`
    msg_file=`mktemp /tmp/cw_$$_email.XXX`
    arc="${B_NAME}-$(date +%s).log.gz"
    core_du=`du -sh ${cod_path}`

    echo "[!] Last modification time: "`stat -c %y ${cod_path}` > $report
    echo "[!] Coredump size: "$core_du >> $report
    echo -e "[!] Command:\n"$cmd >> $report

    cat $trace_file | gzip > $arc && ( \
        echo -e "--${MARKER}\n"
        cat $report
        echo -e $B_START | sed "s/REPLACE_IT/${arc}/"
        cat $arc | openssl base64
        for log_p in $LOGS; do
            [ -r $log_p ] || continue
            echo -e $B_START | sed "s/REPLACE_IT/`basename $log_p`.log.gz/"
            tail -n 10000 $log_p | gzip -f | openssl base64
        done
        echo $B_END
    ) > $msg_file
    rm -f $arc

    # https://st.yandex-team.ru/SEARCHPRODINCIDENTS-607
    for email in $CW_EMAILS; do
        echo "Sending report to $email"
        cat $msg_file | mail -s "${SUBJ}" -a "Content-Type: multipart/mixed; boundary=\"${MARKER}\"" $email
    done
}

get_gdb() {
    [[ -f gdb_toolkit.tgz ]] && {
        [[ -x gdb/bin/gdb ]] || tar xzf gdb_toolkit.tgz &>/dev/null
        [[ -x gdb/bin/gdb ]] || exit_with_error "Cannot unpack gdb_toolkit.tgz"
        echo "gdb/bin/gdb" && return 0
    }
    [[ -x ya ]] && echo "./ya tool gdb" || exit_with_error "No GDB was found. Cannot proceed"
}

verify_defined BSCONFIG_IDIR
verify_defined BSCONFIG_IPORT
[[ -z $LOGS ]] && LOGS='/db/www/logs/*global-rtyserver-'"$BSCONFIG_IPORT"'*'  # Use /db/www/logs by default
[[ -z $CW_INTERVAL ]] && CW_INTERVAL=15   # Process cores created in the last 15 minutes by default
[[ -z $B_NAME ]] && guess=$(basename $BSCONFIG_IDIR/*-$BSCONFIG_IPORT 2>/dev/null) && [[ -r $BSCONFIG_IDIR/$guess ]] && B_NAME=$guess  # Detect default B_NAME if possible
verify_defined B_NAME

#
# Main routine
#
GDB=$(get_gdb) || exit 1
DUMPS_DIR="/cores"
[[ -d ${DUMPS_DIR} ]] || DUMPS_DIR=`dirname $(sysctl -n kernel.core_pattern | tr ' ' '\\n' | grep '^/' | tail -1)`
[[ -d ${DUMPS_DIR} ]] || DUMPS_DIR=`dirname $(sysctl -n kernel.core_pattern | rev | cut -f1 -d" " | rev)`
[[ -d ${DUMPS_DIR} ]] || DUMPS_DIR=`/coredumps`

C_PATH="$BSCONFIG_IDIR/configs/description"
if [[ -f $C_PATH ]]; then
    S_NAME=`get_key service $B_NAME`
    CTYPE=`get_key ctype "no_ctype"`
    # if config exists, but service name is an empty use binary name instead
    S_NAME=`[ $S_NAME ] && echo $S_NAME || $B_NAME`
else
    S_NAME=`get_service_name $B_NAME`
    CTYPE=`get_tag ctype 'no_ctype'`
    [[ "$CTYPE" == "prestable" ]] && CTYPE="stable"  # do not separate sas
fi

CORE_B_NAME=`echo $B_NAME | head -c 15`

#Find cores by B_NAME
CORES=$(find ${DUMPS_DIR}/ -mmin -$CW_INTERVAL -name "*${CORE_B_NAME}*")

#Find cores by pid from crash.log (if available)
if [[ -n $LOOP_SECTION ]]; then
    PIDS=$(cat ../$(basename $(pwd) | sed 's:_[^_]*$::')_*/crash/crash.log | grep 'section='"$LOOP_SECTION" | grep -v 'exit_code=0' \
                 | awk -v FS='\t' -v TS=$(( $(date +%s) - $CW_INTERVAL * 60 ))000 \
                     'function V(k){for(i=1;i<=NF;i++){if (match($i,k"=(.*)",a)){return a[1];}}}'`
                    `'{ if (V("timestamp_ms")>TS) print V("pid"); }' \
                 | head -n 50 | xargs | tr ' ' '|')
    [[ -n $PIDS ]] && CORES=$CORES" "$(find -H ${DUMPS_DIR}/ -mmin -$(($CW_INTERVAL + 50)) -type f | egrep '\.('"$PIDS"')\.[0-9]+$')

    CORES=$(echo $CORES | xargs | tr ' ' \\n | sort -u | xargs)
fi

SLOT_NAME_INTERNAL=`hostname`:${BSCONFIG_IPORT}
SLOT_NAME=${SLOT_NAME:-$SLOT_NAME_INTERNAL}

FREE_SPACE=`df /place | tail -n 1 | awk '{print $4}'`

for cod_path in $CORES; do
    trace_file=`mktemp /tmp/cw_$$_trace.XXX`
    cmd="${GDB} --core=${cod_path} --eval-command='thread apply all bt' --batch ${BSCONFIG_IDIR}/${B_NAME}"
    eval "$cmd 2>&1" > $trace_file
    md5=`tail -50 $trace_file | grep -A 20 "abort" | egrep "^#" | perl -ne 'print $1, "\n" if(m|^#\d+\s+\w+\s(.+)$|);' | md5sum | perl -pe 's/\s+\-$//;'`
    core_du=`du -sh ${cod_path}`
    core_size=`echo ${core_du} | awk '{print $1}'`

    [[ -n $CW_EMAILS ]] && mail_report_to_saas $trace_file $cod_path     # Mail sending is disabled if CW_EMAILS is not set

    cat $trace_file | curl -d @- http://saas-cores.n.yandex-team.ru/corecomes?md5="${md5}&ctype=${CTYPE}&service=${S_NAME}&server=${SLOT_NAME}&time=`date +%s`&corepath=${cod_path}&idir=${BSCONFIG_IDIR}&freespace=${FREE_SPACE}&coresize=${core_size}"
done
