= Bash = == bashrc == {{{#!highlight sh umask 0002 export EDITOR="emacs" export HISTSIZE=100000 export history_control=ignoredups export MANPAGER=less unset LS_COLORS unset TZ # Test if shell is interactive case "$-" in *i*) # Options alias grep='grep --color=auto' alias hg='/appl/mercurial/hg' alias nano='nano -w' # if [[ "${EUID}" == 0 ]] ; then # alias ssh='ssh -CY' # alias s="echo \"Executing su - $@\"; su - $@" # else # alias ssh='ssh -ACY' # fi #alias wget='wget -c' #alias ping='ping -c10' alias ls='ls --color=auto' alias ll='ls -alrt --color' alias dire='ls -lrth' alias ..='cd ..' # Typos alias cd..='cd ..' alias xs='cd' # special hosts alias scgate='ssh scgate' alias schp01='ssh schp01' alias schp02='ssh schp02' alias schp03='ssh schp04' alias schp04='ssh schp04' # abbreviations alias ll='ls -alrt --color' alias e='echo -e' alias p='ps -AlFwww --forest' # Features and functions alias top='unset top_skip; for a in $(eval echo {1..${LINES}}); do top_skip="${top_skip}\n"; done; echo -e ${top_skip}; unset top_skip; top -H -n1000 -c' b(){ unset NUMARGS NONNUMARGS for ARG in $@; do if [[ ${ARG} =~ ^[0-9]+$ ]]; then NUMARGS="${NUMARGS} ${ARG}" else [ "${ARG}" = "-m" ] && NONNUMARGS="${NONNUMARGS} -u all -a" NONNUMARGS="${NONNUMARGS} ${ARG}" fi done if echo "${NONNUMARGS}" | grep -vwq '\-l'; then NONNUMARGS="-w ${NONNUMARGS}" fi if [ "${NONNUMARGS##* }" = '-m' -a -x /opt/LSF/bin/bjobs ]; then # The last non-numeric argument is -m this can only be wrong # Therefore I use ${HOSTNAME} as additional argument if I am on a host where /opt/LSF/bin/bjobs is executable NONNUMARGS="${NONNUMARGS} ${HOSTNAME}" fi ARGS="${NONNUMARGS} ${NUMARGS}" if [ -x /opt/LSF/bin/bjobs ]; then /opt/LSF/bin/bjobs ${ARGS} else ssh scacc1 /opt/LSF/bin/bjobs ${ARGS} fi unset NONNUMARGS NUMARGS ARGS } mycat(){ for FILE in $@; do if [ -r "${FILE}" ]; then if [ "$(awk 'BEGIN{l=0}; {l+=1}; END{print l}' ${FILE})" -gt "$((LINES-LINES/10))" ]; then less ${FILE} else cat ${FILE} fi else echo ERROR: File \""${FILE}"\" is not readable. 1>&2 fi done } # "cd" wrapper that interpretes its argument and extracts the directory automatically # It understands # - . dir dir/file host:dir host:dir/file user@host:dir user@host:dir/file c(){ local arg arg="${1}" # Strip username if echo "${arg}" | grep -q '@' && finger $(echo "${arg}" | awk -F'@' '{print $1}') 2>&1 | grep -v ": no such user." 1>/dev/null; then arg=$(echo "${arg}" | cut -d'@' -f 2-) fi # Strip hostname if echo "${arg}" | grep -q ':' && f_ping $(echo "${arg}" | awk -F':' '{print $1}') 2>/dev/null; then arg=$(echo "${arg}" | cut -d':' -f 2-) fi # Tilde expansion if echo "${arg}" | grep -q '~'; then arg="$(eval ls -d ${arg} 2>/dev/null)" fi # Strip filename if [ -f "${arg}" ]; then arg="$(dirname "${arg}")" fi # Now it should be a directory in most cases if [ -d "${arg}" ]; then cd "${arg}" else cd $@ fi return $? } d(){ du -h --max-depth=1 $@ } f(){ local ARGS if [ $# -eq 0 ]; then ARGS="${USER}" else ARGS="$@" fi finger ${ARGS} 2>&1 | grep -v ": no such user." || grep ${ARGS} /etc/passwd } alias l 2>/dev/null 1>&2 && unalias l l(){ ls -alrt --color $@ | tail -n $((LINES-LINES/10)) } n(){ local NEDITFILES NEDITFILES NEDITWHICH local NEDIT=${NEDIT:=/applbin/nedit} [ -x ${NEDIT} ] || NEDIT="$(which nedit)" [ -x ${NEDIT} ] || NEDIT="/shared/appl/opensoft/Linux/nedit/bin/nedit" for NEDITARG in $@; do if [ -r "${NEDITARG}" ]; then NEDITFILES="${NEDITFILES} ${NEDITARG}" elif which ${NEDITARG} 1>/dev/null 2>&1; then NEDITFILES="${NEDITFILES} $(which ${NEDITARG})" else NEDITFILES="${NEDITFILES} ${NEDITARG}" fi done ${NEDIT} ${NEDITFILES} & } t(){ tail -n $((LINES-LINES/10)) $@ } str(){ for a in $@; do strace -p ${a} 2>&1 | head -n $((LINES-LINES/10)) done } # Time format for debugging f_date(){ date "+%F_%T" } # Funktion, die Keychain startet und SSH_AUTH_SOCK und SSH_AGENT_PID korrekt setzt f_keychain(){ for KC in /usr/bin/keychain ${HOME}/bin/keychain; do if [ -x "${KC}" ]; then break fi done if [ -x "${KC}" ]; then for KEY in ${HOME}/.ssh/id_dsa ${HOME}/.ssh/id_rsa; do if [ -r "${KEY}" ];then eval $(${KC} $@ --lockwait 120 --attempts 10 --timeout 8928 -q --eval ${KEY}) fi done fi } f_ping(){ ping -c1 ${1} 1>/dev/null || ping -c2 ${1} 1>/dev/null || ping -c3 ${1} 1>/dev/null } f_bswitch_error_to_firstqueue(){ if [ ${1} ]; then user=${1} else user=all fi for a in $(bjobs -q error -u ${user} | awk '{if ( $1 ~ /[0-9]/ && $3 ~/PEND/) {print $1}}'); do bswitch $(bhist -l ${a} | gj -ofd=' ' -keys=firstqueue) ${a} done unset user } f_bswitch_from_user_if_ws_pings(){ if [ "${1}" ]; then echo "using ${1} as DEAD_WORKSTATION" DEAD_WORKSTATION="${1}" shift else echo usage: f_bswitch_from_user_if_ws_pings DEAD_WORKSTATION AFFECTED_USER return 1 fi if [ ${1} ]; then AFFECTED_USER="${1}" shift fi a=1 until ping -c2 ${DEAD_WORKSTATION} >/dev/null 2>&1; do sleep ${a} && ((a++)) done && f_bswitch_error_to_firstqueue ${AFFECTED_USER} unset DEAD_WORKSTATION AFFECTED_USER a } # GNU (bash) function from Gentoo (ebuild.sh) f_strip_duplicate_slashes() { if [[ -n $1 ]] ; then local removed=$1 while [[ ${removed} == *//* ]] ; do removed=${removed//\/\///} done echo ${removed} fi } bkill_hanging_job(){ ts_of_youngest_file=0 for a in *; do ts="$(date +%s --date="$(stat -L -c %y ${a})")" && [ "${ts}" -gt "${ts_of_youngest_file}" ] && ts_of_youngest_file=${ts} done age_of_youngest_file="$[($(date +%s)-${ts_of_youngest_file})]" if [ "${age_of_youngest_file}" -lt "$[5*60*60]" ]; then echo sleep $[$[5*60*60]-${age_of_youngest_file}+10] else /shared/appl/tools/beta/vwhpc_debug_job 1>&2 if tail -n1 JOB_OUTPUT.1383924.Radth-Vect-coupling | grep -q 'Waiting for'; then echo bkill it fi fi } # GNU (bash) function from Gentoo (ebuild.sh) unpack() { local srcdir local x local y local myfail local tar_opts="" [ -z "$*" ] && die "Nothing passed to the 'unpack' command" for x in "$@"; do vecho ">>> Unpacking ${x} to ${PWD}" y=${x%.*} y=${y##*.} if [[ ${x} == "./"* ]] ; then srcdir="" elif [[ ${x} == ${DISTDIR%/}/* ]] ; then die "Arguments to unpack() cannot begin with \${DISTDIR}." elif [[ ${x} == "/"* ]] ; then die "Arguments to unpack() cannot be absolute" else srcdir="${DISTDIR}/" fi [[ ! -s ${srcdir}${x} ]] && die "${x} does not exist" myfail="failure unpacking ${x}" case "${x##*.}" in tar) tar xof "${srcdir}${x}" ${tar_opts} || die "$myfail" ;; tgz) tar xozf "${srcdir}${x}" ${tar_opts} || die "$myfail" ;; tbz|tbz2) bzip2 -dc "${srcdir}${x}" | tar xof - ${tar_opts} assert "$myfail" ;; ZIP|zip|jar) unzip -qo "${srcdir}${x}" || die "$myfail" ;; gz|Z|z) if [ "${y}" == "tar" ]; then tar zoxf "${srcdir}${x}" ${tar_opts} || die "$myfail" else gzip -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" fi ;; bz2|bz) if [ "${y}" == "tar" ]; then bzip2 -dc "${srcdir}${x}" | tar xof - ${tar_opts} assert "$myfail" else bzip2 -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" fi ;; 7Z|7z) local my_output my_output="$(7z x -y "${srcdir}${x}")" if [ $? -ne 0 ]; then echo "${my_output}" >&2 die "$myfail" fi ;; RAR|rar) unrar x -idq -o+ "${srcdir}${x}" || die "$myfail" ;; LHa|LHA|lha|lzh) lha xfq "${srcdir}${x}" || die "$myfail" ;; a) ar x "${srcdir}${x}" || die "$myfail" ;; deb) # Unpacking .deb archives can not always be done with # `ar`. For instance on AIX this doesn't work out. If # we have `deb2targz` installed, prefer it over `ar` for # that reason. We just make sure on AIX `deb2targz` is # installed. if type -P deb2targz > /dev/null; then deb2targz "${srcdir}/${x}" || die "$myfail" mv "${srcdir}/${x/.deb/.tar.gz}" data.tar.gz else ar x "${srcdir}/${x}" || die "$myfail" fi ;; lzma) if [ "${y}" == "tar" ]; then lzma -dc "${srcdir}${x}" | tar xof - ${tar_opts} assert "$myfail" else lzma -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" fi ;; *) vecho "unpack ${x}: file format not recognized. Ignoring." ;; esac done # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE # should be preserved. find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w } s(){ if [ -r "${HOME}/.Xauthority" ]; then export XAUTHORITY=${HOME}/.Xauthority_${RANDOM}_${$} cp -pr ${HOME}/.Xauthority ${XAUTHORITY} chmod go+r ${XAUTHORITY} fi if [[ "${EUID}" == 0 ]]; then echo \"Executing su - $@\" su - $@ else if [ -f /opt/vw/local/scr/su_user ]; then if [ -z "${1}" ] || finger "${1}" 2>&1 | grep -q ": no such user." ; then SUDO_USER="$(/opt/LSF/bin/bjobs -w -u all -r -m ${HOSTNAME} 2>/dev/null | awk '/RUN/{print $2; exit}')" echo "No User provided guessing it: ${SUDO_USER}" fi sudo su_user ${SUDO_USER} $@ fi fi rm -f ${XAUTHORITY} unset SUDO_USER XAUTHORITY } #VW commands # ------------------------------------------------ for I in blsload bexec bpwd clocate bhwcheck bres; do eval alias $I=\'ssh scacc1 /opt/vw/bin/$I\' done case $(uname -n) in evebe919|wsntbk02*) for I in blsload bexec bpwd clocate bhwcheck bres; do eval alias $I=\'ssh scacc1 /opt/vw/bin/$I\' done if [ "${USER}" == etorabi ]; then alias ssh="f_keychain; ssh -AY" alias scp="f_keychain; scp -p" alias bsub="ssh scacc1 /opt/LSF/bin/bsub -P0OS3170" fi alias myjgen="~/jgen/bin/jgen" alias myxjgen="~/jgen/bin/xjgen" alias s="su - $@" f_keychain -Q --nolock --noask ;; sc*|scnec*|schp*|evebe700|evebe103) g(){ if [ "$#" -eq "0" ]; then PIDS=$(ps -AlFwww | awk '{if ($2 == "R" && $3 !~ /root/ && $15 !~ /pts/) {print $0 > "/dev/stderr"; print $4}}') else PIDS="$@" fi for PID in ${PIDS}; do taskset -cp ${PID} gstack ${PID} done } # General function that uses variables from the .env file f_mv2ws(){ mv2ws_error="(${USER}@${HOSTNAME}) ERROR in function f_mv2ws:" if [ $# -eq 0 ]; then echo "$(f_date) ${mv2ws_error} a filename to copy needs to be provided" 1>&2 return 1 fi MV2DIRARGS="-v" if [ "${1}" = "-k" ]; then MV2DIRARGS="${MV2DIRARGS} -k" shift fi if [ -z "${JOB_WSOUTDIR:-}" ]; then echo "$(f_date) ${mv2ws_error} The variable JOB_WSOUTDIR is empty." 1>&2 return 2 fi TARGET="${JOB_WSOUTDIR}" if [ $(echo "${JOB_WSOUTDIR:-}" | awk -F: '{print NF}') -eq 1 ]; then if [ -n "${JOB_WSNAME:-}" -a -n "${JOB_WSUSER:-}" ]; then TARGET="${JOB_WSUSER}@${JOB_WSNAME}:${JOB_WSOUTDIR}" else echo "$(f_date) ${mv2ws_error} could not set TARGET correctly" 1>&2 echo "$(f_date) ${mv2ws_error} JOB_WSUSER=\"${JOB_WSUSER}\" JOB_WSNAME=\"${JOB_WSNAME}\" JOB_WSOUTDIR=\"${JOB_WSOUTDIR}\"" 1>&2 return 3 fi fi if [ "${JOB_WSUSESCP}" = 'yes' ]; then MV2DIRARGS="${MV2DIRARGS} -s" fi for FILE in $@; do if [ -r "${FILE}" ]; then [ "${JOB_WSGROUPW}" = 'yes' ] && chmod g+w ${FILE} /opt/vw/bin/mv2dir ${MV2DIRARGS} ${TARGET} ${FILE} fi done unset mv2ws_error MV2DIRARGS TARGET FILE } # backward compatibility cp2ws(){ f_mv2ws -k $@;} f_run_on_ws(){ if [ -z "${JOB_WSNAME}" -o -z "${JOB_WSOUTDIR}" -o -z ${JOB_WSUSER} ]; then echo "A required variable is empty" 1>&2 echo -e "JOB_WSUSER=\"${JOB_WSUSER}\"\tJOB_WSNAME=\"${JOB_WSNAME}\"\tJOB_WSOUTDIR=\"${JOB_WSOUTDIR}\"" 1>&2 return 1 fi if [ $# -eq 0 ]; then echo "cd ${JOB_WSOUTDIR}" ssh ${JOB_WSUSER}@${JOB_WSNAME} else ssh ${JOB_WSUSER}@${JOB_WSNAME} "cd ${JOB_WSOUTDIR}; $@" fi return $? } # "R"un on all "H"osts of this "J"ob f_rhj(){ if [ "$#" -eq "0" ]; then echo "You have to provide a command to execute" 1>&2 fi if [ -z "${HOSTNAME}" -o -z "${JOB_HOSTLIST}" -o -z ${JOB_BATCHDIR} ]; then echo "A required varable is empty" 1>&2 echo -e "HOSTNAME=\"${HOSTNAME}\"\JOB_HOSTLIST=\"${JOB_HOSTLIST}\"\JOB_BATCHDIR=\"${JOB_BATCHDIR}\"" 1>&2 fi # only if the execution if $@ suceeds it will be performed on the other hosts as well if $@; then # Try to figure out if remote noninteractive execution will work # if ALIAS=$(alias ${1}) 2>/dev/null; then # ${1}="${ALIAS} " # fi # unset ALIAS for JOB_HOST in ${JOB_HOSTLIST}; do if [ ${JOB_HOST} != ${HOSTNAME} ]; then ssh -AY ${JOB_HOST} "cd ${JOB_BATCHDIR}; $@" fi done else echo "The command \"$@\" failed to execute locally. No remote execution will be performed" 1>&2 fi } for I in blsload bexec bpwd clocate bhwcheck bres; do eval unalias ${I} done if [ "${USER}" = "etorabi" ]; then alias bsub="bsub -P0OS3170" fi if which bjoblog 2>/dev/null 1>&2; then alias bjob="bjobs" fi JOB_BATCHDIR=$(ls -ld /tmp/BATCH.${USER}.${LSB_JOBID:=*} 2>/dev/null | awk '/\/tmp\/BATCH.'${USER}'/{print $NF; exit}') if [ "${JOB_BATCHDIR}" ]; then if [ -r "${JOB_BATCHDIR}/.env" ]; then # We are on the first node source ${JOB_BATCHDIR}/.env else LSB_JOBID=${LSB_JOBID:=$(ls -ld /tmp/BATCH.${USER}.* 2>/dev/null | awk -F'/' '{print $(NF-1)}' | awk -F'.' '{print $NF; exit}')} #' FIRSTHOST=$(/opt/LSF/bin/bjobs ${LSB_JOBID} | awk '/RUN/ {print $6}' | awk -F'*' '{print $2}') JOB_ENV="/hosts/${FIRSTHOST}/$(echo ${JOB_BATCHDIR} | awk -F'/' '{print $3}')/.env" source ${JOB_ENV} || echo "Could not source JOB_ENV=\"${JOB_ENV}\" . " fi cd ${JOB_BATCHDIR} if [ "${JOB_APPL_FUNCTIONS}" ]; then # RT: source ${JOB_APPL_FUNCTIONS} for interactive usage source ${JOB_APPL_FUNCTIONS} fi fi ;; e?ebe*) for CMD in bacct bhosts bjobs bhist lshosts bstop bresume brequeue bswitch brun bkill bqueues bmod btop bbot bpeek bjdepinfo lsload ; do if [ -x /opt/LSF/bin/${CMD} ]; then eval alias ${CMD}=/opt/LSF/bin/${CMD} else eval alias ${CMD}=\'ssh scacc1 /opt/LSF/bin/${CMD}\' fi done if [ -x /opt/LSF/bin/bsub ]; then alias bsub="/opt/LSF/bin/bsub -P0OS3170" else alias bsub="ssh scacc1 /opt/LSF/bin/bsub -P0OS3170" fi ;; *) ;; esac # recreate the contents of this file by starting as root@cae-support # ssh hpcappl@scacc1 "env ENV=/opt/vw/etc/kshrc KSHRC_IFLAG=1 /bin/ksh -c set" | grep ^SC > /appl/tools/beta/vwhpc.hosts for HOSTLIST in /shared/appl/tools/beta/vwhpc.hosts /appl/tools/beta/vwhpc.hosts ~/.vwhpc.hosts; do if [ -r "${HOSTLIST}" ]; then source ${HOSTLIST} && break fi done SCC="${SC_ML370} $SCC7 $SCC8 $SCC9 ${SC10} ${SCC1} ${SCC2} ${SCC3} ${SCC5} ${SCC4} ${SCC6} ${SCNVH} ${SC_EVEBE}" for I in ${SCC} scacc1; do eval alias ${I}=\'ssh ${I}\' done # if [[ ${EUID} == 0 ]] ; then # export PS1='\[\033[1;33m\]\t \[\033[1m\]\[\033[1;31m\]\h:\[\033[1;34m\]\w \$\[\033[00m\] ' # else # create coloured bash prompt # black background PS1="\[\033[1;40m\]" # yellow time PS1="${PS1}\[\033[1;33m\]\d \t" case ${USER} in fox0fdn) # green PS1="${PS1} \[\033[1;32m\]\u" ;; root) # red PS1="${PS1} \[\033[1;31m\]\u" ;; *) # light blue PS1="${PS1} \[\033[1;36m\]\u" ;; esac # gray PS1="${PS1}\[\033[1;30m\]@" case ${HOSTNAME} in wsntbk02*) # reset to black background and green hostname (no username) PS1="\[\033[1;40m\]\[\033[1;33m\]\t " if [[ "${EUID}" == 0 ]]; then # red PS1="${PS1}\[\033[1;31m\]\h" else # green PS1="${PS1}\[\033[1;32m\]\h" fi ;; evebe919) # green PS1="${PS1}\[\033[1;32m\]\h" # export PATH=~/jgen/bin/:$PATH ;; evebe777|evebe759) # red PS1="${PS1}\[\033[1;31m\]\h" ;; e?ebe*) # white PS1="${PS1}\[\033[1;37m\]\h" ;; e?evm*) # white PS1="${PS1}\[\033[1;35m\]\h" ;; schp0*|scacc1) # light blue PS1="${PS1}\[\033[1;36m\]\h" unset brun bjobs ;; scc*|schp*) # light blue PS1="${PS1}\[\033[1;36m\]\h" ;; *) # pink PS1="${PS1}\[\033[1;35m\]\h" ;; esac export PS1="${PS1}\[\033[1;30m\]:\[\033[1;34m\]\w \$\[\033[00m\] " # fi ;; *) : # This shell is not interactive ;; esac #if [ "${USER}" = "etorabi" -a "${HOSTNAME}" = "evebe700" ]; then # unalias s 2>/dev/null # s(){ # echo "executing \"sudo su - $@ /opt/vw/shared/appl/tools/beta/etorabi\"" 1>&2 # sudo su - $@ /opt/vw/shared/appl/tools/beta/etorabi # } #fi bjur(){ delay='60' [ $# -gt 1 ] && delay="$2" while :; do clear echo "Updated at "`date +%H:%M:%S` bjobs -u $1 sleep $delay done } bjud() { delay='60' [ $# -gt 1 ] && delay="$2" while :; do clear echo "Updated at "`date +%H:%M:%S` bjobs -u $1 -d | tail -n 50 sleep $delay done } bju() { delay='60' [ $# -gt 1 ] && delay="$2" while :; do clear echo "Updated at "`date +%H:%M:%S` bjobs -u $1 echo ".-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-.-'-" bjobs -u $1 -d | tail -n 50 sleep $delay done } bjq() { delay='60' [ $# -gt 1 ] && delay="$2" while :; do clear echo "Updated at "`date +%H:%M:%S` bjobs -q $1 -u all echo bjobs -q $1 -u all -d | tail -n 50 sleep $delay done } # support colors in less export LESS_TERMCAP_mb=$'\E[01;31m' export LESS_TERMCAP_md=$'\E[01;31m' export LESS_TERMCAP_me=$'\E[0m' export LESS_TERMCAP_se=$'\E[0m' export LESS_TERMCAP_so=$'\E[01;44;33m' export LESS_TERMCAP_ue=$'\E[0m' export LESS_TERMCAP_us=$'\E[01;32m' }}}