Shell - Encoding

character encoding

iconv --from-code=ISO-8859-1 --to-code=UTF-8 iso-8859-1.txt > utf-8.txt echo "ä" | recode html..utf8 echo "ü" | w3m -dump -T text/html

Remove Byte-Order Marks (BOM)

sed '1 s/\xEF\xBB\xBF//' < input > output tail --bytes=+4 UTF8WithBom.txt > UTF8WithoutBom.txt

sed with canonical hex

% hexdump -C bla 00000000 c2 96 0a 00000003 % sed 's/\xc2\x96/-/g' bla -

make nice unix names recursively

#!/bin/bash # #///////////////////////////////////////////////# # # # name: mkNiceNames # # # VERSION=0.32 # # # # author: ulf laube # # descr.: build nice unix names recursively # # SYNTAX: mkNiceNames [file|directory] # # # #///////////////////////////////////////////////# #///////////////////////////////////////////////# # global variables # #///////////////////////////////////////////////# SCRIPT=`basename "$0"` SCRIPT_DIR=`dirname "$0"` MACHINE_NAME=${HOSTNAME} DATESTAMP=`date +%y%m%d` TIMESTAMP=`date +%Y%m%d%H%M%S` YEAR=`date +%Y` DEPTH=1 PREFIX='' WRKDIR='.' #///////////////////////////////////////////////# # settings # #///////////////////////////////////////////////# TR_SET='[:alnum:]-_,.' NO_FIRST_CHAR_SET='-_,' NO_LAST_CHAR_SET="${NO_FIRST_CHAR_SET}." #///////////////////////////////////////////////# # main # #///////////////////////////////////////////////# ### dirnames while true;do COUNT=`find -mindepth ${DEPTH} -maxdepth ${DEPTH} -type d | wc -l` if [ $COUNT -eq 0 ];then break fi find -mindepth ${DEPTH} -maxdepth ${DEPTH} -type d | while read FULLNAME;do DIRNAME=`dirname "${FULLNAME}"` NAME=`basename "${FULLNAME}"` NEWNAME=$(echo "${NAME}" | sed -e 's/ /_/g' | iconv -c -f utf8 -t ascii//TRANSLIT | tr -c -d ${TR_SET} | tr -s '_') if [ -n "${NO_FIRST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/^[${NO_FIRST_CHAR_SET}]//") fi if [ -n "${NO_LAST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/[${NO_LAST_CHAR_SET}]$//") fi if [ "${NAME}" != "${NEWNAME}" ];then mv -v "${DIRNAME}/${NAME}" "${DIRNAME}/${NEWNAME}" fi done let DEPTH=DEPTH+1 done ### filenames find -type f | while read FULLNAME;do DIRNAME=`dirname "${FULLNAME}"` SUFFIX=$(echo -e "${FULLNAME}" | tr -d '\r' | tr -d '\n' | sed 's/.*\.//') if [ "$(basename "${SUFFIX}")" != "$(basename "${FULLNAME}")" ];then NAME=`basename "${FULLNAME}" | sed "s/\.${SUFFIX}$//"` if [ "x${NAME}" != "x" ]; then if [ $(echo "${NAME}" | sed 's/^["'\'']//' | cut -c 1) = '.' ]; then PREFIX='.' fi fi NEWSUFFIX=$(echo -e ${SUFFIX} | tr -c -d [:alnum:] | tr [:upper:] [:lower:]) NEWNAME="${PREFIX}$(echo "${NAME}" | sed -e 's/ /_/g' | iconv -c -f utf8 -t ascii//TRANSLIT | tr -c -d ${TR_SET} | tr -s '_').${NEWSUFFIX}" if [ -n "${NO_FIRST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/^[${NO_FIRST_CHAR_SET}]//") fi if [ -n "${NO_LAST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/[${NO_LAST_CHAR_SET}]$//") fi if [ "${NAME}.${SUFFIX}" != "${NEWNAME}" ];then mv -v "${DIRNAME}/${NAME}.${SUFFIX}" "${DIRNAME}/${NEWNAME}" fi else NAME=`basename "${FULLNAME}"` if [ $(echo "${NAME}" | sed 's/^["'\'']//' | cut -c 1) = '.' ]; then PREFIX='.' fi NEWNAME="${PREFIX}$(echo "${NAME}" | sed -e 's/ /_/g' | iconv -c -f utf8 -t ascii//TRANSLIT | tr -c -d ${TR_SET} | tr -s '_')" if [ -n "${NO_FIRST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/^[${NO_FIRST_CHAR_SET}]//") fi if [ -n "${NO_LAST_CHAR_SET}" ];then NEWNAME=$(echo "${NEWNAME}" | sed "s/[${NO_LAST_CHAR_SET}]$//") fi if [ "${NAME}" != "${NEWNAME}" ];then mv -v "${DIRNAME}/${NAME}" "${DIRNAME}/${NEWNAME}" fi fi PREFIX='' done #///////////////////////////////////////////////# # clean up + exit # #///////////////////////////////////////////////# exit 0

VIM

vim encoding

tnx 2 aaron-mueller.de

set encoding

<a href='http://vimdoc.sourceforge.net/htmldoc/options.html#%27encoding%27' class="ext" target="_blank" onfocus="this.blur()" title="">vimdoc</a><p> set encoding=utf-8 <a href='http://unix.stackexchange.com/questions/23389/how-can-i-set-vims-default-encoding-to-utf-8' class="ext" target="_blank" onfocus="this.blur()" title="">tnx 2 unix.stackexchange.com</a> Or instead, set the LANG environment variable to indicate that UTF-8 is your preferred character encoding. This will affect not just Vim but any software which relies on LANG to determine how it should represent text. For example, to indicate that text should appear in English (en), as spoken in the United States (US), encoded as UTF-8 (utf-8), set LANG=en_US.utf-8.

fileencoding

<a href='http://vimdoc.sourceforge.net/htmldoc/options.html#%27fileencodings%27' class="ext" target="_blank" onfocus="this.blur()" title="">vimdoc</a> fileencoding=utf-8

search multiple consecutive spaces

tnx 2 stackoverflow.com

/[^ ]\zs \+

Void Match

tnx 2 aaron-mueller.de

" hint :help pattern-overview " switch on :highlight needLineBreak ctermbg=gray :match needLineBreak /.\%>80v/ " switch off :match none

Perform a non-regex search/replace in vim

tnx 2 spektom

set nomagic

clear last search highlighting

tnx 2 Shaun Bouckaert

:let @/ = ''

vim syntax highlighting of OBO Flat File

The <a href='http://www.geneontology.org/GO.format.obo-1_2.shtml' class="ext" target="_blank" onfocus="this.blur()" title="OBO Flat File Format">OBO Flat File Format</a> is a text file format to store ontologies.

/opt/oc/share/vim/syntax/obo.vim

" Vim syntax file " Language: OBO Flat File Format " URL: http://www.geneontology.org/GO.format.obo-1_4.shtml " Maintainer: Ulf Laube <ulf dab laube at ulaube dab de> " Last Change: 2014-06-06 " Version: 1.4 if exists("b:current_syntax") finish endif setlocal iskeyword+=: syn match oboTrailMod " \[.*\]$" syn match oboTrailMod " {.*}$" syn match oboStanzaTerm "^\[.*\]" syn match oboStanzaTag "^.\{-}:" syn match oboStanzaSynValue "\".*\"" syn match oboHeader "^format-version:" syn match oboHeader "^data-version:" syn match oboHeader "^ontology:" syn match oboHeader "^date:" syn match oboHeader "^saved-by:" syn match oboHeader "^auto-generated-by:" syn match oboHeader "^subsetdef:" syn match oboHeader "^import:" syn match oboHeader "^synonymtypedef:" syn match oboHeader "^idspace:" syn match oboHeader "^default-relationship-id-prefix:" syn match oboHeader "^id-mapping:" syn match oboHeader "^remark:" syn match oboRem "!.*$" highlight link oboStanzaTerm Identifier highlight link oboStanzaTag Type highlight link oboStanzaSynValue Special highlight link oboTrailMod Statement highlight link oboRem Comment highlight link oboHeader Constant let b:current_syntax = "obo"

/etc/vimrc

... au BufRead,BufNewFile *.obo set filetype=obo ...

/etc/vimrc.local

"#///////////////////////////////////////////////# "# # "# name: vimrc.local # "# # "VERSION=0.24 # "# # "# author: ulf dab laube at ulaube dab de # "# # "# descr.: vim configuration # "# # "# SYNTAX: use file with this lines in vimrc: # "# # "# if filereadable(expand("/etc/vimrc.local")) # "# source /etc/vimrc.local # "# endif # "# # "#////////////+//////////////////////////////////# "# changelog: # "# # "# 20140606 new key mappings # "# 20120912 change hex to F6 # "# 20120912 change F9 to line wrap switch # "# 20120223 encoding added # "# 20120703 f7 autoindent # "# # "#////////////+//////////////////////////////////# " ### set default encoding set encoding=utf-8 "set bomb " ### set bell modes set noerrorbells set vb t_vb="<Esc>|f" " ### enable syntax highlighting syntax on " ### enable modline detection like # vim: ft=sh set modeline " ### automatically indent lines (default) set noautoindent " ### number of whitspaces in tabs set tabstop=4 " ### expand tab2whitspaces "set expandtab set shiftwidth=4 " ### autospace for brackets "set cindent set autoindent "set smartindent " ### select case-insenitiv search (not default) set ignorecase set smartcase " ### highlighting all search matches set hlsearch "set incsearch " ### set default line highlighting set cursorline "set cursorcolumn " ### enable linenumbering set number " ### set list characters " <ctr>-k : vr hh , FB , ?S "set lcs=tab:├─,eol:█,trail:▓ " <ctr>-k : vr hh , PI , ?S set lcs=tab:├─,eol:¶,trail:▓ " ### syntax highlighting ### " /opt/oc/share/vim/syntax/obo.vim au BufRead,BufNewFile *.obo set filetype=obo " ### mappings ### " map <F2> : split window noremap <silent> <F2> :split!<CR> " map <F3> : vsplit window noremap <silent> <F3> :vsplit!<CR> " map <F4> : highlighting from column 80 to end of line highlight needLineBreak ctermbg=gray let g:highlightLineBreak = 0 function HighlightingNeedLineBreak() if g:highlightLineBreak == 1 :match none let g:highlightLineBreak = 0 return endif :match needLineBreak /.\%>80v/ let g:highlightLineBreak = 1 endfunction noremap <silent> <F4> :call HighlightingNeedLineBreak() <CR> " map <F5> : show cursorcolumn on/off noremap <silent> <F5> :set cursorcolumn!<CR> " map <F6> : nomagic search on/off noremap <silent> <F6> :set nomagic!<CR> " map <F7> : autoindent on/off noremap <silent> <F7> :set autoindent!<CR> " map <F8> : readonly on/off noremap <silent> <F8> :set ro!<CR> " map <F9> : line wrap on/off noremap <silent> <F9> :set invwrap!<CR> " map <F10> : syntax highlighting on/off map <F10> :if has("syntax_items")<CR>syntax off<CR>else<CR>syntax on<CR>endif<CR><CR> " map <F11> : list characters on/off noremap <silent> <F11> :set list!<CR> " map <F12> : linenumbers on/off noremap <silent> <F12> :set number!<CR> " map <shift><F1> : show key mappings noremap <silent> <S-F1> :echo system("echo '### key mappings ###';grep '^\" map' /etc/vimrc.local")<CR> " map <shift><F10> : clear last search highlighting noremap <silent> <S-F10> :let @/ = ''<CR> " map <shift><F12> : hex on/off function Fxxd() let c=getline(".") if c =~ '^[0-9a-f]\{7}:' :%!xxd -r else :%!xxd endif endfunction map <S-F12> :call Fxxd()<CR>

save highlighted syntax in a HTML file

tnx 2 Maciej Bliziński

:runtime! syntax/2html.vim or :TOhtml

sed

eliminating HTML-tags

sed -e 's/<[^>]*>//g' text.html

center all text in the middle of 79-column width

tnx 2 sed

sed -e :a -e 's/^.\{1,77\}$/ & /;ta'

tick marks

tnx 2 freebsd.org

echo "bla'' bla'" | sed -e 's/'\'\''/'\''/g'

add 'foo' to end of each line

tnx 2 stackoverflow.com

sed 's/$/ foo/' file

add text to the beginning of a file

tnx 2 Dennis Williams

sed -i '1s/^/task goes here\n/' todo.txt

replace new line character

tnx 2 linux.dsplabs.com.au

sed ':a;N;$!ba;s/\n/\t/g' foo

eliminating special filename characters

sed 's/[\<\>\|\*\?\(\)\$\`\´\!\;\#\^\/\"\{\}\[\]]//g'

remove last character of a file

cat file | sed '$s/.$//'

print last character of a file

sed -n '$s/.*\(.\)$/\1/p' file

Shell - Bash

crop a URL from 'a href' html tag

grep -o -E 'href="([^"#]+)"' <FILE> | cut -d\" -f 2

while loop with counter

COUNTER=5 while true;do echo $COUNTER if [ $COUNTER -eq 0 ];then break fi let COUNTER=COUNTER-1 done echo "finished"

bash arrays

bash arrays I

while read i;do LABELS=`echo -e "$i"| cut -f1` oIFS="$IFS" IFS=';' #IFS=$(echo -en "\t") TERMS=($LABELS) IFS="$oIFS" TERMS_NBRS=$[ ${#TERMS[@]} -1 ] #echo "+++$LABELS+++" #echo ${TERMS[@]} #echo ${#TERMS[@]} for j in `seq 0 ${TERMS_NBRS}` ;do echo -e "${TERMS[$j]}" done ### OR ### for j in ${TERMS[*]};do echo "${j}" done done

bash arrays II

REVERSE_DIRS=(a b) for i in ${REVERSE_DIRS[@]}; do echo $i;done

bash hashes / bash associative arrays

tnx 2 linuxjournal tnx 2 bash-hackers

declare -A aa #aa["hello world"]="from bash" id="hello world" aa["${id}"]="from bash" echo -e "${aa[${id}]}" for KEY in "${!aa[@]}"; do echo $KEY ${aa[$KEY]}; done

seq A - Z

printf "%s\n" {A..Z}

yes / no

echo -n "Installer: Adminrechte werden benötigt. Wollen sie fortfahren?: " read answer case "$answer" in Yes|yes|Y|y|"") echo "Ok los geht's" ;; No|no|N|n) echo "Abbruch." exit 1 ;; *) echo "Unbekannter Parameter" ;; esac

escape apostrophes in sql statements

a=$(echo "$i" | sed s/"'"/"\\\'"/g)

ascii spinner

tnx 2 vodkamelone.de

while true; do for i in '|' '/' '-' '\'; do echo -n $i; sleep 0.25; echo -ne '\b'; done; done

day diff

tnx 2 ureader.de

DATUM1=20121026 DATUM2=20120718 echo $(( ($(date -d "$DATUM1" +%s) - $(date -d "$DATUM2" +%s)) / 86400 ))

shift

a=$1 echo "arg1 ; $a" echo "all args : $@" shift a=$1 echo "shited arg to 1 ; $a" echo "all args : $@"

Print duplicate lines

tnx 2 unix.com

awk '$0==s{print s RS $0}{s=$0}' file

Getting the count of unique values in a column

tnx 2 stackoverflow.com

cut -f 2 file | sort | uniq -c | sort -nr cut -f 2 file | sort | uniq -c | sort -nr | sed 's/^\s*//' | grep -v '1 '

week of month

WEEK_OF_MONTH=`eval echo '$((' $(date '+%-W - $(date -d %Y/%m/01 "+%%-W")' ) '+1 ))'`

Skip comment lines

tnx 2 Дмитрий Маликов

while read i;do [[ "${i}" =~ ^#.*$ ]] && continue done

stop_at_end_of_a_time.sh

#!/bin/bash function timetest () { #t_ist=`date +%H%M` echo -en "$i t_start : $t_start / t_ist : $t_ist / t_end : $t_end\t" if [ $t_ist -eq $t_end ];then echo -en "STOPPED" fi if [ $t_start -lt $t_end ];then if [ $t_ist -gt $t_end ];then echo -en "STOPPED" else if [ $t_start -gt $t_ist ];then echo -en "STOPPED" fi fi else if [ $t_ist -gt $t_end -a $t_ist -lt $t_start ];then echo -en "STOPPED" fi fi echo "" } function tests () { i=1; t_start=0300;t_end=0600 timetest i=2; t_start=0300;t_end=1700 timetest i=3; t_start=0500;t_end=0600 timetest i=4; t_start=0600;t_end=1700 timetest i=5; t_start=1100;t_end=0600 timetest i=6; t_start=1100;t_end=1700 timetest i=7; t_start=1800;t_end=0600 timetest i=8; t_start=1800;t_end=1700 timetest } t_ist=0301;tests t_ist=0801;tests t_ist=1201;tests t_ist=1801;tests t_ist=2301;tests

get total number of cpus/cores in server

tnx 2 Corona688

awk '/^processor/ { N++} END { print N }' /proc/cpuinfo

bash: preserve leading/trailing tabs and whitespaces while reading/writing line by line

while IFS='' read -r i;do echo "$i" done<foo>bar

Modifying variable inside while loop is not remembered

tnx 2 Stack Overflow

<![CDATA[ while read line; do ... done <<EOT $(echo -e $lines) EOT ]]>

make nested directory structure from 00 - FF

tnx 2 that other guy

printf "%02x\n" {0..255}|while read i;do printf "%02x\n" {0..255}|while read j;do mkdir -p "$i/$j";done;done

sort huge files

<![CDATA[ sort -n --parallel=16 --buffer-size=10G foo > bar ]]>