#!/bin/bash # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This script converts all db files of a cyrus installation from their # existing format to the format required by the current installation. # The format of current db files is determined using the 'file' command # with a magic file added for skiplist db, the new format is read from # a config file usually in /usr/share/cyrus-imapd/rpm/db.cfg, which is # created while compiling. After converting, the db.cfg file is # copied to a cache file usually at /var/lib/imap/rpm/db.cfg.cache to # allow bypassing this converting script if both files are identical. # While this is a bit less secure, it may be useful on big server where # db converting is done automatically. # # This script can safely be run as root, it will reexec itself as user # cyrus if needed. # # author: Simon Matter, Invoca Systems # changelog # v1.0.1, Oct 22 2002 Simon Matter # - added two-step conversion method # # v1.0.2, Jan 10 2003 Simon Matter # - fixed a bug where cvt_cyrusdb was called to convert empty or # nonexistent files # # v1.0.3, Mar 14 2003 Simon Matter # - fixed a problem with new versions of the file command # # v1.0.4 # - added GPL license # # v1.0.5, May 02 2003 Simon Matter # - modified exec path # # v1.0.6, Jul 18 2003 Simon Matter # - changed db3 to berkeley # - added new db backends for 2.2 # # v1.0.7, Jan 23 2004 Simon Matter # - included some modifications from Luca Olivetti # - added masssievec functionality # # v1.0.8, Jan 28 2004 Simon Matter # - convert sieve scripts to UTF-8 before calling masssievec # # v1.0.9, Jan 29 2004 Simon Matter # - convert sieve scripts to UTF-8 only if sievec failed before # # v1.0.10, Feb 24 2004 Simon Matter # - change su within init script to get input from # /dev/null, this prevents hang when running in SELinux # # v1.0.11, Mar 02 2004 Simon Matter # - fixed SELinux fix # # v1.0.12, Dec 16 2004 Simon Matter # - use runuser instead of su if available # # v1.0.13, Jul 15 2005 Simon Matter # - don't use flat in the two step conversion, use skiplist instead # # v1.0.14, Jul 18 2005 Simon Matter # - replace the order of the magic files in the file call to make # sure skiplist is detected correctly. # # v1.0.15, Aug 17 2005 Simon Matter # - add functionality to export all berkeley db files to skiplist # # v1.1.0, Aug 18 2005 Simon Matter # - fix export functionality, try to recover Berkeley databases # as much as possible before any conversion. # # v1.1.1, Dec 05 2005 Simon Matter # - run db_checkpoint in background with a timeout to prevent # that cyrus-imapd doesn't start at all if it hangs. # # v1.1.2, Dec 06 2005 Simon Matter # - make handling of db_checkpoint more robust # # v1.2.0, Jan 12 2006 Simon Matter # - adopt for cyrus-imapd-2.3 # # v1.2.1, Jan 13 2006 Simon Matter # - code cleanup # # v1.2.2, Nov 29 2007 Simon Matter # - add ability to handle "@include" options in imapd.conf, patch # provided by Tim Bannister # # v1.2.3, Feb 07 2008 Simon Matter # - add ability to handle tabs in imapd.conf, patch provided # by Franz Knipp # - disable default values for some config options like sievedir # # v1.2.4, Apr 23 2008 Simon Matter # - add support for statuscache.db # # v1.3.0, Sep 29 2008 Simon Matter # - add multi-instance support # # v1.3.1, Oct 09 2008 Simon Matter # - improve variable handling # # v1.3.2, May 26 2009 Simon Matter # - add some sanity checks to multi-instance support # # v1.3.3, May 27 2009 Simon Matter # - make some cosmetic changes # # v1.3.4, Dec 22 2009 Simon Matter # - add support for user_deny.db # # v1.3.5, Jul 28 2010 Simon Matter # - fix $RUNUSER call for cyrus user having /sbin/nologin as shell # # v1.3.6, Jan 12 2011 Simon Matter # - fix multi-instance support # # v1.3.7, Jan 13 2011 Simon Matter # - run BDB recovery only if we have BDB support compiled in # - convert from BDB only if we have BDB support compiled in # # v1.4.0, Jan 03 2013 Simon Matter # - add support for custom database paths # - add support for persistent databases on tmpfs # # v1.4.1, Jan 04 2013 Simon Matter # - add support for {configdirectory} VERSION=1.4.1 PIDFILE=/var/run/cyrus-master${INSTANCE}.pid # instance config CYRUSCONF=/etc/cyrus${INSTANCE}.conf IMAPDCONF=/etc/imapd${INSTANCE}.conf # make sure what we have is a valid instance # and that config files are present if [ -n "$INSTANCE" ]; then [ -L /etc/rc.d/init.d/cyrus-imapd${INSTANCE} ] || exit 1 fi [ -f $CYRUSCONF ] || exit 1 [ -f $IMAPDCONF ] || exit 1 if [ -f $PIDFILE ]; then read CYRUS_PID < $PIDFILE if [ -n "$CYRUS_PID" ]; then if ps -p $CYRUS_PID > /dev/null 2>&1; then echo "ERROR: cyrus-master is running, unable to convert databases!" exit 1 fi fi fi if [ ! -f $IMAPDCONF ]; then echo "ERROR: configuration file '${IMAPDCONF}' not found, exiting!" exit 1 fi # fallback to su if runuser not available if [ -x /sbin/runuser ]; then RUNUSER=runuser else RUNUSER=su fi # force cyrus user for security reasons if [ $(whoami) != "cyrus" ]; then exec $RUNUSER - cyrus -s /bin/bash -c "cd $PWD < /dev/null ; INSTANCE=$INSTANCE $0 $@" fi # special function for migration EXPORT=$1 # files get mode 0600 umask 0166 # show version info in log files echo "cvt_cyrusdb_all version: $VERSION" # expand_config # handle "@include" sections from imapd style config file expand_config() { while read line; do if printf "%s\n" "${line}" | grep -q '^@include:'; then expand_config "$( printf "%s\n" "${line}" | cut -d : -f 2- | sed -e 's/^[\t ]*//' )" else printf "%s\n" "${line}" fi done < $1 } # get_config [] # extracts config option from config file get_config() { searchstr=$1 if config="$(expand_config $IMAPDCONF | egrep "^${searchstr}:")"; then CFGVAL="$(printf "%s\n" "$config" | cut -d : -f 2- | sed -e 's/^[\t ]*//')" else CFGVAL="$2" fi echo "get_config ${1}: \"${CFGVAL}\"" 1>&2 echo "$CFGVAL" } # where to find files and directories data_dir=/usr/share/cyrus-imapd/rpm lib_dir=/usr/lib/cyrus-imapd system_magic=$(file --version | awk '/magic file/ {print $4}') cyrus_magic=${data_dir}/magic cvt_cyrusdb=${lib_dir}/cvt_cyrusdb sievec=${lib_dir}/sievec masssievec=${lib_dir}/masssievec configdirectory=$(get_config configdirectory) || exit 1 sieve_dir=$(get_config sievedir) || exit 1 db_cfg=${data_dir}/db.cfg db_current=${configdirectory}/rpm/db.cfg.current db_cache=${configdirectory}/rpm/db.cfg.cache build_cfg=${data_dir}/build.cfg # source builtin config options [ -f $db_cfg ] || exit 1 . $db_cfg [ -f $build_cfg ] || exit 1 . $build_cfg # overwrite builtins with options from config file while read line; do eval "${line}=\$(get_config ${line} \$${line}) || exit 1" done < <(cut -d"=" -f1 $db_cfg) # cache current config options while read line; do echo "${line}=\"$(eval echo \$${line})\"" done < <(cut -d"=" -f1 $db_cfg) > $db_current # file_type file_type() { this_type=$(file -b -m "${cyrus_magic}:${system_magic}" "$1" 2> /dev/null) if echo "$this_type" | grep -qi skip > /dev/null 2>&1; then echo skiplist elif echo "$this_type" | grep -qi text > /dev/null 2>&1; then echo flat else echo berkeley fi } # expands vars like {configdirectory} expand_vars() { line="$@" line="${line//\{configdirectory\}/${configdirectory}}" echo "$line" } # cvt_file [] cvt_file() { dbfile="$1" new_db="$2" custom="$3" # expand {configdirectory} in paths dbfile="$(expand_vars "$dbfile")" custom="$(expand_vars "$custom")" # handle custom db paths if [ -n "$custom" ]; then if [ ! -s "$custom" ]; then if [ -s "$dbfile" ]; then oldumask=$(umask) umask 0077 mkdir -p "$(dirname "$custom")" umask "$oldumask" cp -a "$dbfile" "$custom" target="$custom" fi fi fi if [ -n "$target" ]; then target="$dbfile" fi if [ -s "$target" ]; then old_db=$(file_type "$target") if [ "$old_db" != "$new_db" ]; then if [ "$old_db" = "berkeley" ]; then if [ "$with_bdb" != "1" ]; then echo "ERROR: database $target is of type berkeley but we have no BDB support!" echo "To use this you have to build cyrus-imapd with BDB support, exiting now." exit 1 fi fi # the two-step conversion is paranoia against the filenames being encoded # inside the database or logfiles (berkeley does this, for example). rm -f "${target}.skiplist" if [ "$old_db" = "skiplist" ]; then cp -a "$target" "${target}.skiplist" else $cvt_cyrusdb -C $IMAPDCONF "$target" "$old_db" "${target}.skiplist" skiplist fi RETVAL=$? ERRVAL=$(( $ERRVAL + $RETVAL )) if [ $RETVAL -eq 0 ]; then rm -f "$target" if [ -s "${target}.skiplist" ]; then if [ "$new_db" = "skiplist" ]; then cp -a "${target}.skiplist" "$target" else $cvt_cyrusdb -C $IMAPDCONF "${target}.skiplist" skiplist "$target" "$new_db" fi fi RETVAL=$? ERRVAL=$(( $ERRVAL + $RETVAL )) if [ $RETVAL -eq 0 ]; then rm -f "${target}.skiplist" else echo "ERROR: unable to convert ${target}.skiplist from skiplist to $new_db" fi else echo "ERROR: unable to convert $target from $old_db to skiplist" fi fi fi if [ "$EXPORT" = "export" ]; then if [ -n "$custom" ]; then mv -f "$custom" "${dbfile}.NEW" mv -f "${dbfile}.NEW" "$dbfile" fi fi } # cvt_to_utf8 cvt_to_utf8() { target="$1" if [ -s "$target" ]; then if ! $sievec -C $IMAPDCONF "$target" "${target}.sievec"; then iconv --from-code=ISO-8859-1 --to-code=UTF-8 --output="${target}.UTF-8" "$target" if [ -s "${target}.UTF-8" ]; then # preserve timestamp touch --reference="${target}" "${target}.UTF-8" mv -f "${target}.UTF-8" "$target" else ERRVAL=$(( $ERRVAL + 1 )) fi fi rm -f "${target}.sievec" fi } ERRVAL=0 if [ "$with_bdb" = "1" ]; then # make sure our Berkeley databases are in a sane state # wait for db_checkpoint to end successfully or kill it after a timeout db_checkpoint -v -1 -h ${configdirectory}/db & DB_CHECK_PID=$! CNT=0 while [ $CNT -lt 60 ]; do if ! kill -0 $DB_CHECK_PID > /dev/null 2>&1; then break fi sleep 1 let CNT+=1 done if kill -0 $DB_CHECK_PID > /dev/null 2>&1; then kill -USR1 $DB_CHECK_PID > /dev/null 2>&1 sleep 1 kill -KILL $DB_CHECK_PID > /dev/null 2>&1 wait $DB_CHECK_PID > /dev/null 2>&1 fi # do a normal recovery db_recover -v -h ${configdirectory}/db RETVAL=$? if [ $RETVAL -ne 0 ]; then # try a catastrophic recovery instead of normal recovery db_recover -v -c -h ${configdirectory}/db RETVAL=$? ERRVAL=$(( $ERRVAL + $RETVAL )) if [ $RETVAL -ne 0 ]; then echo "ERROR: catastrophic recovery of Berkeley databases failed" fi fi fi if [ "$EXPORT" = "export" ]; then # convert all db files to portable format for migration # TODO: quota_db, we don't touch it for now cvt_file "${configdirectory}/deliver.db" "skiplist" "$duplicate_db_path" cvt_file "${configdirectory}/mailboxes.db" "skiplist" "$mboxlist_db_path" cvt_file "${configdirectory}/tls_sessions.db" "skiplist" "$tlscache_db_path" cvt_file "${configdirectory}/annotations.db" "skiplist" "$annotation_db_path" cvt_file "${configdirectory}/ptclient/ptscache.db" "skiplist" "$ptscache_db_path" cvt_file "${configdirectory}/statuscache.db" "skiplist" "$statuscache_db_path" cvt_file "${configdirectory}/user_deny.db" "flat" "$userdeny_db_path" rm -vf ${configdirectory}/db/log.* rm -vf ${configdirectory}/db/__db.* else # always convert db files which have been converted to skiplist # TODO: quota_db, we don't touch it for now cvt_file "${configdirectory}/deliver.db" "$duplicate_db" "$duplicate_db_path" cvt_file "${configdirectory}/mailboxes.db" "$mboxlist_db" "$mboxlist_db_path" cvt_file "${configdirectory}/tls_sessions.db" "$tlscache_db" "$tlscache_db_path" cvt_file "${configdirectory}/annotations.db" "$annotation_db" "$annotation_db_path" cvt_file "${configdirectory}/ptclient/ptscache.db" "$ptscache_db" "$ptscache_db_path" cvt_file "${configdirectory}/statuscache.db" "$statuscache_db" "$statuscache_db_path" cvt_file "${configdirectory}/user_deny.db" "$userdeny_db" "$userdeny_db_path" # do we have to convert all databases? if ! cmp -s $db_current $db_cache; then # we treat sieve scripts the same way like db files find ${sieve_dir}/ -name "*.script" -type f | while read db_file trash; do cvt_to_utf8 "$db_file" done $masssievec $sievec $IMAPDCONF # convert all db files left find ${configdirectory}/user/ -name "*.seen" -type f | while read db_file trash; do cvt_file "$db_file" "$seenstate_db" done find ${configdirectory}/user/ -name "*.sub" -type f | while read db_file trash; do cvt_file "$db_file" "$subscription_db" done find ${configdirectory}/user/ -name "*.mboxkey" -type f | while read db_file trash; do cvt_file "$db_file" "$mboxkey_db" done fi fi # update the config cache file so we can check whether something has changed if [ $ERRVAL -eq 0 ]; then mv -f $db_current $db_cache else rm -f $db_cache rm -f $db_current fi exit $ERRVAL