#!/bin/sh
#
# Replacement for check-vm using (almost) explicit package lists for
# each distro-version-architecture combination
#
LANG=C.UTF-8

_usage()
{
    echo >&2 "Usage: $0 [options]"
    echo >&2 
    echo >&2 "  -A arch      override platform arch"
    echo >&2 "  -D distro    override platform distro"
    echo >&2 "  -d           debug and keep tmp.* files"
    echo >&2 "  -c           check package list"
    echo >&2 "  -i tag       include only lines with a tag"
    echo >&2 "  -m           list only missing packages"
    echo >&2 "  -n           report my package list file and quit"
    echo >&2 "  -s           strip cpan() and pip3() wrappers"
    echo >&2 "  -u           update => -c + fixups"
    echo >&2 "  -U           -u but show me (no changes to package list)"
    echo >&2 "  -v           verbose (extra debugging)"
    echo >&2 "  -V version   override platform version"
    echo >&2 "  -x tag       exclude lines with a tag"
    exit 1
}

_fmt()
{
    if fmt -g "$1" -w "$1" </dev/null >/dev/null 2>&1
    then
	# GNU fmt(1) with -g
	__args="-g $1 -w $1"
    elif fmt "$1" "$1" </dev/null >/dev/null 2>&1
    then
	# FreeBSD fmt(1) with goal [maximum]
	__args="$1 $1"
    else
	# don't know, punt on original fmt(1)
	__args="-w $1"
    fi
    fmt $__args
}

# on entry we have ...
# $tmp.list - packages from package list, with optional -x and -i processing
# 	done
# $tmp.allpkgs - all available packages for this platform
#
_do_check()
{
    # dredge manifest for ALL packages for this distro
    #

    # First get any required packages
    #
    _parse_file $home/other-packages/require \
    | sed >$tmp.require \
	-e 's/{//g' \
	-e 's/}//g' \
    # end
    [ -s $tmp.require ] && $very_verbose && echo >&2 "require: `cat $tmp.require`"

    # Then any skipped packages
    #
    _parse_file $home/other-packages/skip \
    | sed >$tmp.skip \
	-e 's/{//g' \
	-e 's/}//g' \
    # end

    # the optional "skip.<hostname>" file records packages that may be available
    # but for various reasons should be be installed on a particular host ...
    #
    _hostname=`hostname -s`
    if [ -f $home/other-packages/skip.$_hostname ]
    then
	sed <$home/other-packages/skip.$_hostname >>$tmp.skip -e 's/#.*//' -e '/^[ 	]*$/d'
    fi
    [ -s $tmp.skip ] && $very_verbose && echo >&2 "skip: `cat $tmp.skip`"

    # And any unavailable packages
    #
    _parse_file $home/other-packages/unavailable \
    | sed >$tmp.na \
	-e 's/{//g' \
	-e 's/}//g' \
    # end
    [ -s $tmp.na ] && $very_verbose && echo >&2 "unavailable: `cat $tmp.na`"

    # And then process the manifest.
    # The packaging types (tags @ start of manifest lines) we know about.
    #
    cat <<End-of-File >$tmp.tags
dpkg	Ubuntu Debian LinuxMint MX
rpm	RHEL Fedora CentOS openSUSE SUSE SLES OpenMandriva-Lx AmazonLinux OracleLinux
emerge	Gentoo
pkgin	NetBSD
pkg_add	OpenBSD
F_pkg	FreeBSD
S_pkg	OpenIndiana
slackpkg	Slackware
pacman	ArchLinux
brew	Darwin
End-of-File
    pkg_mgr_tag=`sed -n <$tmp.tags -e 's/	/ /g' -e 's/$/ /' -e "/ $distro /s/ .*//p"`
    if [ -z "$pkg_mgr_tag" ]
    then
	echo >&2 "Botch: can't get manifest package manager tag for \"$distro\""
	exit
    fi
    $verbose && echo >&2 "Info: pkg_mgr_tag=$pkg_mgr_tag"
    $verbose && echo >&2 "Info: distro_tag=$distro_tag"

    # sed below needs to ...
    # - remove text to the right of # comment token
    # - remove text after lineno and up to packages, so between : and [
    # - remove trailing ]
    # - strip (...) e.g. (QA optional) and cpan() and pip3()
    # - strip "base <os> install"
    #
    rm -f $tmp.matched
    awk <$home/other-packages/manifest '{ print NR ":" $0 }' \
	| grep -E ":($pkg_mgr_tag|$distro_tag)\?" \
    | sed \
	-e 's/[ 	]*#.*//' \
	-e 's/:[^[]*\[/ /' \
	-e 's/\([a-z][a-z0-9]*\)(\([^)]*\))/\1|\2|/g' \
	-e 's/ *([^)]*)//g' \
	-e 's/\([a-z][a-z0-9]*\)|\([^|]*\)|/\1(\2)/g' \
	-e 's/{[^}]*}//' \
	-e 's/] *$//' \
	-e 's/ *$/ /' \
	-e '/ cpan([^)]*) /{
:morecpan
s// /
t morecpan
}' \
	-e '/ pip3([^)]*) /{
:morepip3
s// /
t morepip3
}' \
	-e 's/ or / /g' \
	-e 's/base .* install//' \
	-e 's/  *$//' \
    | while read lineno pkgs
    do
	$very_verbose && echo >&2 "manifest:$lineno pkgs=\"$pkgs\""
	# some of these are not even real packages ...
	#
	case "$pkgs"
	in
	    N/A\ on\ *|N/A)		continue
			    		;;
	    *SKIP*)			continue
			    		;;
	    *'base FreeBSD install'*)	continue
					;;
	    *'base NetBSD install'*)	continue
					;;
	    *'base OpenBSD install'*)	continue
					;;
	    *'base MacOS install'*)	continue
					;;
	    '')				# comes from cpan(...) alone, so no real pkgs
					continue
					;;
	esac
	rm -f $tmp.havereq
	for pkg in $pkgs
	do
	    for req_pkg in `cat $tmp.require`
	    do
		if echo "$pkg" | grep "^$req_pkg$" >/dev/null
		then
		    # one of the packages mentioned on this manifest line
		    # is required, so assume that's a tie-breaker amongst
		    # the alternatives
		    #
		    touch $tmp.havereq
		    break
		fi
	    done
	done

	# if at least one package from the manifest line is in
	# the list and available, then we're done
	#
	rm -f $tmp.ok
	for pkg in $pkgs
	do
	    [ "$pkg" = "N/A" ] && continue
	    if [ "$pkg" = '?' ]
	    then
		echo "manifest:$lineno $pkgs: unknown package name"
		# not OK, but rest of checking makes no sense so move right along
		#
		touch $tmp.ok
		continue
	    fi
	    pkg_re=`echo "$pkg" | sed -e 's;/;\\\\/;g'`
	    if awk -F '|' <$tmp.list '
BEGIN		{ sts=1 }
		{ for (i = 1; i <= NF; i++) {
		    if ($i == "'"$pkg"'" || $i ~ /^'"$pkg_re"'$/) {
			print $0 >>"'$tmp.matched'"
			sts=0
			# keep going because there may be more than one package
			# in the package list that matches the pattern in the
			# manifest, e.g. may need to have 2 versions of a pkg
		    }
		  }
		}
END		{ exit sts }'
	    then
		# included in list, is it available?
		#
		avail="`grep "^$pkg\$" <$tmp.allpkgs`"
		if [ -z "$avail" ]
		then
		    if $update
		    then
			if [ "$pkg_type" = brew ]
			then
			    echo "$pkg brew" >>$tmp.del
			else
			    echo "$pkg" >>$tmp.del
			fi
		    else
			echo "manifest:$lineno $pkg: in package list, but not available?"
		    fi
		else
		    touch $tmp.ok
		    # keep going so we check all the pkg in $pkgs from the
		    # manifest
		fi
	    fi
	done
	if [ ! -f $tmp.ok ]
	then
	    # no match from the list, so reprocess the packages in the
	    # manifest line looking for available packages as candidates
	    # to be added to the list
	    #
	    for pkg in $pkgs
	    do
		# not in list, unless it is to be skipped, or not available
		# or we have a required package for this manifest line,
		# is it available?
		#
		rm -f $tmp.skip_me
		for skip_pkg in `cat $tmp.skip`
		do
		    skip_pkg_re=`echo "$skip_pkg" | sed -e 's;/;\\\\/;g'`
		    #debug# echo "pkg=$pkg skip_pkg=$skip_pkg skip_pkg_re=$skip_pkg_re" 
		    if echo "$pkg" | awk '
BEGIN		{ sts=1 }
		{ for (i = 1; i <= NF; i++) {
		    if ($i == "'"$skip_pkg"'" || $i ~ /^'"$skip_pkg_re"'$/ || "'"$skip_pkg"'" ~ $i) {
			sts=0
			exit
		    }
		  }
		}
END		{ exit sts }'
		    then
			$very_verbose && echo >&2 "$pkg ($skip_pkg): skipped"
			touch $tmp.skip_me
			break
		    fi
		done
		rm -f $tmp.na_me
		for na_pkg in `cat $tmp.na`
		do
		    na_pkg_re=`echo "$na_pkg" | sed -e 's;/;\\\\/;g'`
		    if echo "$pkg" | awk '
BEGIN		{ sts=1 }
		{ for (i = 1; i <= NF; i++) {
		    if ($i == "'"$na_pkg"'" || $i ~ /^'"$na_pkg_re"'$/) {
			sts=0
			exit
		    }
		  }
		}
END		{ exit sts }'
		    then
			$very_verbose && echo >&2 "$pkg ($na_pkg): not available"
			touch $tmp.na_me
			break
		    fi
		done
		if [ ! -f $tmp.skip_me -a ! -f $tmp.na_me ]
		then
		    if [ ! -f $tmp.havereq ]
		    then
			avail="`grep "^$pkg\$" <$tmp.allpkgs`"
			installed="`sed -e 's/ .*//' <$tmp.installed | grep "^$pkg\$"`"
			if [ -n "$avail" ]
			then
			    if [ "$pkg" = "$avail" ]
			    then
				if $update
				then
				    # special hack for perl-Foo-Bar, don't add
				    # if Foo::Bar cpan already in package list
				    #
				    case "$pkg"
				    in
					perl-*)
					    xxx=`echo "$pkg" | sed -e 's/perl-//' -e 's/-/::/g'`
					    if grep -q "^$xxx cpan" $tmp.orig >/dev/null
					    then
						$verbose && echo >&2 "Info: skip missing $pkg because $xxx cpan already in package list"
						pkg=''
					    fi
					    ;;
				    esac
				    if [ -n "$pkg" ]
				    then
					if [ "$pkg_type" = brew ]
					then
					    echo "$pkg brew" >>$tmp.add
					else
					    echo "$pkg" >>$tmp.add
					fi
				    fi
				else
				    if [ "$avail" = "$installed" ]
				    then
					echo "manifest:$lineno $pkg: not in package list, but already installed?"
				    else
					echo "manifest:$lineno $pkg: not in package list, but available?"
				    fi
				fi
			    else
				navail=`echo "$avail" | wc -w | sed -e 's/ //g'`
				if [ "$navail" -eq 1 ]
				then
				    if $update
				    then
					if [ "$pkg_type" = brew ]
					then
					    echo "$avail brew" >>$tmp.add
					else
					    echo "$avail" >>$tmp.add
					fi
				    else
					if [ "$avail" = "$installed" ]
					then
					    echo "manifest:$lineno $pkg: not in package list, but $avail already installed?"
					else
					    echo "manifest:$lineno $pkg: not in package list, but $avail available?"
					fi
				    fi
				elif [ "$navail" -le 2 ]
				then
				    # don't try already installed here ... it is too hard!
				    #
				    echo "manifest:$lineno $pkg: not in package list, but `echo "$avail" | tr '\012' ' ' | sed -e 's/ $//' -e 's/ / and /'` available?"
				else
				    echo "manifest:$lineno $pkg: not in package list, but `echo "$avail" | tr '\012' ' ' | sed -e 's/ .* \([^ ][^ ]*\) / ... \1/'` available?"
				fi
			    fi
			fi
		    fi
		fi
	    done
	fi
    done

    # are the required ones all there?
    #
    for pkg in `cat $tmp.require`
    do
	if grep -E "(^$pkg )|(^$pkg$)" <$tmp.list >/dev/null
	then
	    : OK
	else
	    if $update
	    then
		if [ "$pkg_type" = brew ]
		then
		    echo "$pkg brew" >>$tmp.add
		else
		    echo "$pkg" >>$tmp.add
		fi
	    else
		echo "$pkg: required but not in package list"
	    fi
	fi
    done

    # and are all the packages in the package list really available?
    #
    sed <$tmp.list -e '/cpan(/d' -e '/pip3(/d' \
    | tr '|' '\012' \
    | while read pkg
    do
	avail="`grep "^$pkg\$" <$tmp.allpkgs`"
	if [ -z "$avail" ]
	then
	    if $update
	    then
		if [ "$pkg_type" = brew ]
		then
		    echo "$pkg brew" >>$tmp.del
		else
		    echo "$pkg" >>$tmp.del
		fi
	    else
		echo "$pkg: in package list but not available"
	    fi
	fi
    done

    # and does the package list contain lines that do not match the
    # manifest?
    if [ -s $tmp.matched ]
    then
	sed <$tmp.list -e '/cpan(/d' -e '/pip3(/d' \
	| sort -o $tmp.list
	sort -o $tmp.matched $tmp.matched
	comm -23 $tmp.list $tmp.matched >$tmp.no-manifest
	if [ -s $tmp.no-manifest ]
	then
	    echo "In package list, but not in manifest ..."
	    _fmt 72 <$tmp.no-manifest | sed -e 's/^/    /'
	fi
    fi

    # any duplicates in the package list are also a problem
    #
    sort $tmp.list | uniq -c \
    | sed -e '/^ *1 /d' -e 's/^ *[0-9][0-9]* //' >$tmp.dup
    if [ -s $tmp.dup ]
    then
	echo "Duplicated in package list ..."
	_fmt 72 <$tmp.dup | sed -e 's/^/    /'
    fi

    # and for rpm and dpkg packaging we have additional package requirements
    # for building and installing from the "spec" and "control" files
    #
    if [ "$pkg_type" = rpm ]
    then
	if ! $home/check-rpm >$tmp.check-rpm 2>&1
	then
	    if $update
	    then
		cat $tmp.check-rpm \
		| while read line
		do
		    case "$line"
		    in
			# Error: libpfm4 in rpm spec but not in package list
			*\ in\ rpm\ spec\ but\ not\ *)
			    echo "$line" | awk '{ print $2 }' >>$tmp.add
			    ;;
			# Error: perl-LWP-UserAgent is not available but LWP::UserAgent cpan is not in the package list
			*\ not\ available\ *\ cpan\ is\ not\ *)
			    echo "$line" | awk '{ print $7,$8 }' >>$tmp.add
			    ;;
			*)
			    echo "$line"
			    ;;
		    esac
		done
	    else
		sed -n <$tmp.check-rpm -e '/Error:/s//rpm-spec:/p'
	    fi
	fi
    fi
    if [ "$pkg_type" = dpkg ]
    then
	if ! $home/check-dpkg >$tmp.check-dpkg 2>&1
	then
	    echo TODO
	    cat $tmp.check-dpkg
	fi
    fi
}

LC_COLLATE=POSIX; export LC_COLLATE

verbose=false
very_verbose=false
debug=false
include=''
exclude=''
missing=false
check=false
nflag=false
strip=false
update=false
show_me=false
while getopts 'A:cdD:i:mnsuUvV:x:?' p
do
    case "$p"
    in
	A)	Arch="$OPTARG"
		;;

	c)	check=true
		;;

	d)	debug=true
		;;

	D)	Distro="$OPTARG"
		;;

	i)	if [ -n "$exclude" ]
		then
		    echo >&2 "Error: -x and -i are mutually exclusive"
		    exit
		fi
		include="$include $OPTARG"
		;;

	m)	missing=true
		;;

	n)	nflag=true
		;;

	s)	strip=true
		;;

	u)	check=true
		update=true
		;;

	U)	check=true
		update=true
		show_me=true
		;;

	v)	if $verbose
		then
		    very_verbose=true
		else
		    verbose=true
		fi
		;;

	V)	Version="$OPTARG"
		;;

	x)	if [ -n "$include" ]
		then
		    echo >&2 "Error: -i and -x are mutually exclusive"
		    exit
		fi
		exclude="$exclude $OPTARG"
		;;

	?)	_usage
		# NOTREACHED
    esac
done
shift `expr $OPTIND - 1`
[ $# -eq 0 ] || _usage

status=1		# failure is the default
if $debug
then
    tmp=`pwd`/tmp
    trap "echo exit sts=\$status; exit \$status" 0 1 2 3 15
else
    tmp=${TMPDIR:-/tmp}/list-packages-$$
    trap "rm -f $tmp.*; exit \$status" 0 1 2 3 15
fi
rm -f $tmp.*

# Need directory where this script is located so we can find the other
# scripts and control files
#
home=`echo $0 | sed -e 's/\/*list-packages$//'`
if [ ! -f $home/whatami ]
then
    echo >&2 "Botch: \$0=$0 -> bad \$home=$home ?"
    exit 1
fi

if [ ! -f $home/packages.rc ]
then
    echo >&2 "Botch: cannot find $home/packages.rc"
    exit
fi

. $home/packages.rc

_setversions
# $distro, $version $full_version, $arch and $full_arch now set
$debug && echo "Debug: distro=$distro version=$version arch=$arch"

_get_distro_tag || exit
_get_pkg_type || exit

# optionally over-ridden from the comand line ...
#
if [ -n "$Distro" ]
then
    $debug && echo >&2 "Debug: forcing distro=$Distro (not $distro)"
    distro="$Distro"
fi
if [ -n "$Version" ]
then
    $debug && echo >&2 "Debug: forcing version=$Version (not $version)"
    version="$Version"
fi
if [ -n "$Arch" ]
then
    $debug && echo >&2 "Debug: forcing arch=$Arch (not $arch)"
    arch="$Arch"
fi

_setup_pkg_list || exit

if $nflag
then
    echo "$home/package-lists/${distro}+${version}+${arch}"
    status=0
    exit
fi

if [ -n "$exclude" ]
then
    $very_verbose && [ -n "$exclude" ] && echo >&2 "Excluding tags:$exclude"
    echo 's/$/ /' >$tmp.sed
    for pkg in $exclude
    do
	echo "/ $pkg /d" >>$tmp.sed
    done
    sed -f $tmp.sed <"$home/package-lists/${distro}+${version}+${arch}"
elif [ -n "$include" ]
then
    $very_verbose && [ -n "$include" ] && echo >&2 "Including only tags:$include"
    echo 's/$/ /' >$tmp.sed
    for pkg in $include
    do
	echo "/ $pkg /p" >>$tmp.sed
    done
    sed -n -f $tmp.sed <"$home/package-lists/${distro}+${version}+${arch}"
else
    cat "$home/package-lists/${distro}+${version}+${arch}"
fi >$tmp.list
cp $tmp.list $tmp.orig

if $missing || $check
then
    # Base on distro info, generate a list of currently installed packages
    # in $tmp.installed and a list of all available packages in $tmp.allpkgs
    #
    rm -f $tmp.installed $tmp.allpkgs
    rm -f $tmp.debug
    case "$distro"
    in
	Ubuntu|Debian|LinuxMint|MX)
	    dpkg-query -W -f '${package} ${status}\n' \
	    | sed -n -e '/ installed$/s/ .*//p' >$tmp.installed
	    apt-cache dumpavail | sed -n -e '/^Package: /s///p' >$tmp.allpkgs
	    ;;

	RHEL|Fedora|CentOS|openSUSE|SUSE\ SLES|OpenMandriva-Lx|AmazonLinux|OracleLinux)
	    rpm -qa --qf '%{NAME}\n' >$tmp.installed
	    if which zypper >/dev/null 2>&1
	    then
		zypper --no-refresh packages 2>&1 \
		| awk -F \| ' { print $3 }' \
		| sed >$tmp.allpkgs \
		    -e 's/  *//g' \
		# end
	    elif which dnf5 >/dev/null 2>&1
	    then
		( dnf5 list --available ; dnf5 list --installed ) 2>/dev/null \
		| sed \
		    -e '/^Available packages/d' \
		    -e '/^Installed packages/d' \
		| awk '{ print $1 }' \
		| sed \
		    -e 's/\.[^.]*$//' \
		| sort \
		| uniq >$tmp.allpkgs
	    elif which dnf >/dev/null 2>&1
	    then
		# note using dnf's system cache, so no metadata refresh,
		# but -C is borked sometimes ...
		#
		for dnf_args in "-C" ""
		do
		    echo "=== dnf $dnf_args list" >>$tmp.debug
		    sudo dnf $dnf_args list 2>&1 \
		    | tee -a $tmp.debug \
		    | sed >$tmp.allpkgs \
			-e '1,/^[^ ]* [pP]ackages/d' \
			-e 's/ .*//' \
			-e 's/\.[^.]*$//' \
		    # end
		    [ -s $tmp.allpkgs ] && break
		done
	    elif which yum >/dev/null 2>&1
	    then
		# note using yum's system cache, so no metadata refresh
		#
		( yum -C list available; yum -C list installed ) 2>&1 \
		| sed >$tmp.allpkgs \
		    -e '1,/^[^ ]* Packages/d' \
		    -e 's/ .*//' \
		    -e 's/\.[^.]*$//' \
		# end
	    fi
	    ;;

	Gentoo)
	    equery list '*' | sed -e 's/.*].*] //' -e 's/-[0-9].*//' >$tmp.installed
	    # TODO $tmp.allpkgs
	    ;;

	NetBSD)
	    pkgin list | sed -e 's/-[0-9].*//' >$tmp.installed
	    pkgin avail | sed -e 's/-[0-9].*//' >$tmp.allpkgs
	    ;;

	OpenBSD)
	    pkg_info -a | sed -e 's/-[0-9].*//' >$tmp.installed
	    # expect lines like this ...
	    # <tr>...<a href="libsysstat-0.4.1p1.tgz">libsysstat-0.4.1p1.tgz</a>...
	    #
	    url="`cat /etc/installurl`/`uname -r`/packages/`uname -m`/"
	    $debug && echo "Debug: fetching list of available packages from $url ..."
	    curl -s $url \
	    | sed -n >$tmp.allpkgs \
		-e '/a href=".*tgz">/{
s/\.tgz<\/a>.*/.tgz/
s/.*>//
s/-[0-9].*//
p
}'
	    ;;

	FreeBSD)
	    pkg info -a | sed -e 's/-[0-9].*//' >$tmp.installed
	    pkg search -L name . | sed -e 's/ .*//' >$tmp.allpkgs
	    ;;

	OpenIndiana)
	    pkg list -H | sed -e 's/ .*//' >$tmp.installed
	    pkg list -a | sed -e 's/ .*//' >$tmp.allpkgs
	    ;;

	Slackware)
	    ls /var/log/packages | sed -e 's/-[0-9].*//' >$tmp.installed
	    # lines after awk look like ...
	    # ./aspell-word-lists/aspell-cs-20040614_1-x86_64-5.txz
	    # ./n/openldap-client-2.4.42-x86_64-1.txz
	    #
	    for f in /var/lib/slackpkg/slackware64-filelist.gz \
		     /var/lib/slackpkg/extra-filelist.gz
	    do
		[ -f "$f" ] || continue
		zcat "$f" \
		| awk '{ print $1 }' \
		| sed \
		    -e 's/\-[0-9].*//' \
		    -e 's;.*/;;' \
		# end
	    done >$tmp.allpkgs
	    ;;

	ArchLinux)
	    pacman -Qi | sed -n -e '/^Name /s/.* : //p' >$tmp.installed
	    pacman -Ss . \
	    | sed -n >$tmp.allpkgs -e '/^[^ ]/s/ .*//p'
	    ;;

	Darwin)
	    brew list --formula >$tmp.installed
	    brew list --cask >>$tmp.installed
	    ls /Applications | sed -e 's/\.app$//' >>$tmp.installed
	    # Note: limited to homebrew packages ... those that could be
	    #       installed in /Applications is too hard to find
	    #
	    brew formulae >$tmp.allpkgs
	    brew casks >>$tmp.allpkgs
	    ;;

    esac
    if ( $missing || $check ) && [ ! -f $tmp.installed ]
    then
	echo >&2 "Botch: can't generate tmp.installed for $distro"
	exit
    fi
    if [ -s $tmp.allpkgs ]
    then
	sort <$tmp.allpkgs | uniq >$tmp.tmp
	mv $tmp.tmp $tmp.allpkgs
    else
	[ -s $tmp.debug ] && cat >&2 $tmp.debug
	[ -f $tmp.allpkgs ] && ls -l $tmp.allpkgs >&2
	echo >&2 "Botch: can't generate tmp.allpkgs for $distro"
	exit
    fi
fi

# map cpan and pip3 tags => cpan(...) and pip3(...) and brew(...)
#
sed <$tmp.list >$tmp.tmp \
    -e 's/$/ /' \
    -e 's/\(.*\) cpan/cpan(\1)/' \
    -e 's/\(.*\) pip3/pip3(\1)/' \
    -e 's/ $//' \
# end
mv $tmp.tmp $tmp.list

# do optional `cmd` expansion
#
if grep '`' $tmp.list >/dev/null
then
    rm -f $tmp.tmp
    cat $tmp.list | while read pkg
    do
	case "$pkg"
	in
	    *'`'*'`'*)
			eval echo $pkg >>$tmp.tmp
			;;
	    *)
	    		echo "$pkg" >>$tmp.tmp
			;;
	esac
    done
    mv $tmp.tmp $tmp.list
fi

# strip comments, remaining tags, white space and empty lines
#
sed -n <$tmp.list >$tmp.tmp \
    -e 's/#.*//' \
    -e 's/[ 	]*|[ 	]*/|/g' \
    -e 's/[ 	].*//' \
    -e 's/[ 	][ 	]*//g' \
    -e '/./p' \
# end
mv $tmp.tmp $tmp.list

if $check
then
    _do_check
    if $update
    then
	# updates?
	#
	cp "$home/package-lists/${distro}+${version}+${arch}" $tmp.new
	if [ -s $tmp.dup ]
	then
	    if $verbose
	    then
		echo >&2 "Info: duplicate packages to be fixed ..."
		_fmt 72 <$tmp.dup | sed -e 's/^/    /'
	    fi
	    # simply delete and add
	    #
	    cat $tmp.dup >>$tmp.del
	    cat $tmp.dup >>$tmp.add
	fi
	if [ -s $tmp.del ]
	then
	    if $verbose
	    then
		echo >&2 "Info: packages to be deleted ..."
		_fmt 72 <$tmp.del | sed -e 's/^/    /'
	    fi
	    sort $tmp.del | uniq \
	    | while read pkg
	    do
		awk <$tmp.new >$tmp.tmp '
$0 == "'"$pkg"'"	{ next }
			{ print }'
		mv $tmp.tmp $tmp.new
	    done
	fi
	if [ -s $tmp.add ]
	then
	    if $verbose
	    then
		echo >&2 "Info: packages to be added ..."
		_fmt 72 <$tmp.add | sed -e 's/^/    /'
	    fi
	    # add in sorted order (or close to that) ... insertion sort
	    # in the first candidate spot
	    #
	    sort $tmp.add | uniq \
	    | while read pkg
	    do
		awk <$tmp.new >$tmp.tmp '
/^#/			{ print; next }
done == 0 && $1 > "'"$pkg"'"		{ print "'"$pkg"'"; done = 1 }
			{ print }
END			{ if (done == 0) print "'"$pkg"'" }'
		mv $tmp.tmp $tmp.new
	    done
	fi
	if ! diff -q "$home/package-lists/${distro}+${version}+${arch}" $tmp.new >/dev/null
	then
	    if $verbose || $show_me
	    then
		echo "Diffs ..."
		diff -u "$home/package-lists/${distro}+${version}+${arch}" $tmp.new
	    fi
	    if ! $show_me
	    then
		cp $tmp.new "$home/package-lists/${distro}+${version}+${arch}"
	    fi
	else
	    $show_me && echo "No package list updates."
	fi
    fi
    status=0
    exit
fi

if $missing
then
    # special handling for cpan() and pip3() packages
    #
    $very_verbose && cp $tmp.list $tmp.list.before
    rm -f $tmp.sed
    grep '^cpan(' $tmp.list \
    | sed -e 's/cpan(//' -e 's/)//' \
    | while read module
    do
	if echo "use $module;" | perl >/dev/null 2>&1
	then
	    echo "/cpan($module)/d" >>$tmp.sed
	fi
    done
    grep '^pip3(' $tmp.list \
    | sed -e 's/pip3(//' -e 's/)//' \
    | while read module
    do
	if which python3 >/dev/null 2>&1
	then
	    python=python3
	elif which pmpython >/dev/null 2>&1
	then
	    python=pmpython
	else
	    python=''
	    $very_verbose && echo >&2 "Warning cannot file python"
	fi
	if [ -n "$python" ]
	then
	    if $python -c "import $module" >/dev/null 2>&1
	    then
		echo "/pip3($module)/d" >>$tmp.sed
	    fi
	fi
    done
    if [ -f $tmp.sed ]
    then
	sed -f $tmp.sed <$tmp.list >$tmp.tmp
	mv $tmp.tmp $tmp.list
	if $very_verbose
	then
	    echo >&2 "Diffs after cpan() and pip3() mangling ..."
	    diff $tmp.list.before $tmp.list
	fi
    fi

    if [ "$distro" = "ArchLinux" ]
    then
	# special handling for Arch Linux (pacman) naming of packages
	# pacman -Qi (for tmp.installed) => gcc
	# pacman -Ss (for tmp.allpkgs) => core/gcc
	# and our package list (tmp.list) uses the longer form ... so cull
	# the prefix on the assumption that the probability of a name
	# collision is really small, but then we need to do the matching
	# the hard way to generate a list with the full package names
	#
	mv $tmp.list $tmp.tmp
	touch $tmp.list
	cat $tmp.tmp \
	| while read full_pkg
	do
	    pkg=`echo $full_pkg | sed -e 's;^.*/;;'`
	    if grep "^$pkg\$" $tmp.installed >/dev/null
	    then
		:
	    else
		echo "$full_pkg" >>$tmp.list
	    fi
	done
    else
	# strip all packages in both $tmp.list and $tmp.installed and
	# leave the result in $tmp.list
	#
	tr '|' '\012' <$tmp.list | sort | uniq >$tmp.1
	sort <$tmp.installed | uniq >$tmp.2
	if $very_verbose
	then
	    echo >&2 "Required:"
	    _fmt 72 <$tmp.1 | sed -e 's/^/    /' >&2
	    echo >&2 "Installed:"
	    _fmt 72 <$tmp.2 | sed -e 's/^/    /' >&2
	fi
	comm -23 $tmp.1 $tmp.2 >$tmp.tmp
	# now we may have alternates installed ...
	#
	rm -f $tmp.newlist
	touch $tmp.newlist
	cat $tmp.tmp \
	| while read missing
	do
	    if grep "^$missing|" $tmp.list >$tmp.out
	    then
		# the primary package is missing ... if any of the
		# alternates are present (not missing) then we're OK
		#
		rm -f $tmp.ok
		cut -d '|' -f 2- <$tmp.out \
		| tr '|' '\012' \
		| sed -e '/^$/d' \
		| while read alt
		do
		    if grep "^$alt$" $tmp.2 >/dev/null
		    then
			$verbose && echo >&2 "Info: missing: $missing OK because alternate $alt installed"
			touch $tmp.ok
			break
		    fi
		done
		[ -f $tmp.ok ] || echo  "$missing" >>$tmp.newlist
	    elif grep -E "\|$missing($|\|)" $tmp.list >$tmp.out
	    then
		# this is an alternate ... only report the primary
		# (first on line) package if it is also missing
		#
		:
	    else
		echo  "$missing" >>$tmp.newlist
	    fi
	done
	mv $tmp.newlist $tmp.list
    fi
fi

# report manifest missing package name (?:<lineno>) on stderr
#
if grep '^?:[0-9][0-9]*$' $tmp.list >$tmp.tmp
then
    ( tr '\012' ' ' <$tmp.tmp ; echo ) >&2
    grep -v '^?:[0-9][0-9]*$' $tmp.list >$tmp.tmp
    mv $tmp.tmp $tmp.list
fi

# list missing packages (if any) on a single line on stdout
#
if [ -s $tmp.list ]
then
    if $strip
    then
	sed -e 's/cpan(//g' -e 's/pip3(//g' -e 's/)//g' <$tmp.list
    else
	cat $tmp.list
    fi \
    | ( tr '\012' ' ' ; echo )
fi

status=0
